19:39:27.107 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
19:39:27.122 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task9OVAZw/test-tmp
19:39:27.125 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:39:27.134 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task9OVAZw/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:39:27.452 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
19:39:27.602 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:39:27.603 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:39:27.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:27.609385 4621 mini_oidc.cc:152] Starting JWKS server
19:39:27.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:27.614540 4621 webserver.cc:460] Webserver started at http://127.0.0.1:38673/ using document root <none> and password file <none>
19:39:27.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:27.618446 4621 mini_oidc.cc:189] Starting OIDC Discovery server
19:39:27.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:27.621894 4621 webserver.cc:460] Webserver started at http://127.0.0.1:34407/ using document root <none> and password file <none>
19:39:27.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:39:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:39:27.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:39:27Z Disabled control of system clock
19:39:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:27.682122 4621 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/wal
19:39:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/data
19:39:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/logs
19:39:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/data/info.pb
19:39:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.126:35933
19:39:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.126
19:39:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:27.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:27.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:27.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:27.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:27.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:27.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38673/jwks.json with env {}
19:39:28.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.009739 4633 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:28.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.010320 4633 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:28.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.010908 4633 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.048169 4633 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.048487 4633 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.048683 4633 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.048862 4633 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.049175 4633 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.049328 4633 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38673/jwks.json
19:39:28.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:28.084173 4633 master_runner.cc:386] Master server non-default flags:
19:39:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:28.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/data
19:39:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/wal
19:39:28.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.126:35933
19:39:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38673/jwks.json
19:39:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/data/info.pb
19:39:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.126
19:39:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:28.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/logs
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:28.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:28.085896 4633 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:28.088203 4633 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:28.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.109166 4639 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:29.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.109620 4642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:29.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:28.110896 4640 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:29.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:28.111176 4633 server_base.cc:1034] running on GCE node
19:39:29.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.370289 4633 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:29.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.373904 4633 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:29.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.375883 4633 hybrid_clock.cc:648] HybridClock initialized: now 1790624369375824 us; error 94 us; skew 500 ppm
19:39:29.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.376878 4633 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:29.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.385088 4633 webserver.cc:460] Webserver started at http://127.4.131.126:33141/ using document root <none> and password file <none>
19:39:29.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.386169 4633 fs_manager.cc:359] Metadata directory not provided
19:39:29.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.386377 4633 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:29.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.386816 4633 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.392139 4633 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/data/instance:
19:39:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a4624ac66ce4728b2df68d3c391f367"
19:39:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:29 on dist-test-slave-htqb"
19:39:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.393886 4633 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/wal/instance:
19:39:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a4624ac66ce4728b2df68d3c391f367"
19:39:29.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:29 on dist-test-slave-htqb"
19:39:29.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.404398 4633 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.011s sys 0.000s
19:39:29.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.410861 4649 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.412086 4633 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
19:39:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.412461 4633 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/wal
19:39:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a4624ac66ce4728b2df68d3c391f367"
19:39:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:29 on dist-test-slave-htqb"
19:39:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.412886 4633 fs_report.cc:383] FS layout report
19:39:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/wal
19:39:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/wal
19:39:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/data/data
19:39:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:29.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.454877 4633 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:29.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.456662 4633 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:29.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.457197 4633 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.534992 4633 rpc_server.cc:307] RPC server started. Bound to: 127.4.131.126:35933
19:39:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.535080 4700 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.131.126:35933 every 8 connection(s)
19:39:29.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.538069 4633 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/data/info.pb
19:39:29.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.543813 4701 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:39:29.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.548889 4621 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 4633
19:39:29.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.549455 4621 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-0/wal/instance
19:39:29.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.554996 4621 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/wal
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/data
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/logs
19:39:29.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/data/info.pb
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:29.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.125:34735
19:39:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.125
19:39:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:29.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:29.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38673/jwks.json with env {}
19:39:29.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.562196 4701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } has no permanent_uuid. Determining permanent_uuid...
19:39:29.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.590685 4701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } has no permanent_uuid. Determining permanent_uuid...
19:39:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.592967 4653 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.131.125:34735: connect: Connection refused (error 111)
19:39:29.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.596827 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.125:34735: Network error: Client connection negotiation failed: client connection to 127.4.131.125:34735: connect: Connection refused (error 111)
19:39:29.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.646344 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } attempt: 1
19:39:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.651049 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.125:34735: Network error: Client connection negotiation failed: client connection to 127.4.131.125:34735: connect: Connection refused (error 111)
19:39:29.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.719543 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } attempt: 2
19:39:29.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.723308 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.125:34735: Network error: Client connection negotiation failed: client connection to 127.4.131.125:34735: connect: Connection refused (error 111)
19:39:29.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.814684 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } attempt: 3
19:39:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.818210 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.125:34735: Network error: Client connection negotiation failed: client connection to 127.4.131.125:34735: connect: Connection refused (error 111)
19:39:29.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.857853 4703 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:29.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.858417 4703 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:29.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.858966 4703 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:29.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.890511 4703 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:29.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.890775 4703 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:29.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.890954 4703 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:29.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.891196 4703 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:29.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.891393 4703 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:29.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.891517 4703 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38673/jwks.json
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.925477 4703 master_runner.cc:386] Master server non-default flags:
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/data
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/wal
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.125:34735
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38673/jwks.json
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/data/info.pb
19:39:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.125
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/logs
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.926628 4703 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:29.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.928730 4703 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:29.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.950619 4713 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:29.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.961740 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } attempt: 4
19:39:29.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.966182 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.125:34735: Network error: Client connection negotiation failed: client connection to 127.4.131.125:34735: connect: Connection refused (error 111)
19:39:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:30.265749 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } attempt: 5
19:39:30.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:30.272152 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.125:34735: Network error: Client connection negotiation failed: client connection to 127.4.131.125:34735: connect: Connection refused (error 111)
19:39:30.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:30.819736 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } attempt: 6
19:39:30.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:30.828908 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.125:34735: Network error: Client connection negotiation failed: client connection to 127.4.131.125:34735: connect: Connection refused (error 111)
19:39:31.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.950737 4712 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:31.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:29.950767 4715 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:31.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:29.952167 4703 server_base.cc:1034] running on GCE node
19:39:31.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.170325 4703 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.172941 4703 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:31.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.174314 4703 hybrid_clock.cc:648] HybridClock initialized: now 1790624371174268 us; error 58 us; skew 500 ppm
19:39:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.175398 4703 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:31.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.182660 4703 webserver.cc:460] Webserver started at http://127.4.131.125:42645/ using document root <none> and password file <none>
19:39:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.183769 4703 fs_manager.cc:359] Metadata directory not provided
19:39:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.184021 4703 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.184530 4703 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.189663 4703 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/data/instance:
19:39:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22cca2e2f29742b19b5cd3085a2776e5"
19:39:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:31 on dist-test-slave-htqb"
19:39:31.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.190912 4703 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/wal/instance:
19:39:31.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22cca2e2f29742b19b5cd3085a2776e5"
19:39:31.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:31 on dist-test-slave-htqb"
19:39:31.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.197983 4703 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
19:39:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.203189 4724 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:31.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.204089 4703 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
19:39:31.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.204406 4703 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/wal
19:39:31.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "22cca2e2f29742b19b5cd3085a2776e5"
19:39:31.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:31 on dist-test-slave-htqb"
19:39:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.204735 4703 fs_report.cc:383] FS layout report
19:39:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/wal
19:39:31.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/wal
19:39:31.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/data/data
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:31.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:31.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.240374 4703 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.241784 4703 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.242237 4703 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.312496 4703 rpc_server.cc:307] RPC server started. Bound to: 127.4.131.125:34735
19:39:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.312582 4775 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.131.125:34735 every 8 connection(s)
19:39:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.315773 4703 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/data/info.pb
19:39:31.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.321056 4776 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:39:31.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.325629 4621 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 4703
19:39:31.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.326054 4621 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-1/wal/instance
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.328939 4621 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/wal
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/data
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/logs
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/data/info.pb
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.124:38727
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.124
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:31.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38673/jwks.json with env {}
19:39:31.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.336054 4776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } has no permanent_uuid. Determining permanent_uuid...
19:39:31.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.359248 4776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } has no permanent_uuid. Determining permanent_uuid...
19:39:31.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.374686 4776 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } has no permanent_uuid. Determining permanent_uuid...
19:39:31.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.377113 4728 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.381510 4776 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.431082 4776 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 1
19:39:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.435627 4776 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.504181 4776 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 2
19:39:31.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.508071 4776 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:31.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.599591 4776 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 3
19:39:31.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.603530 4776 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.653395 4778 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.653935 4778 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.654584 4778 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.683058 4778 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.683344 4778 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.683511 4778 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:31.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.683667 4778 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:31.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.683837 4778 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:31.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.684044 4778 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38673/jwks.json
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.716943 4778 master_runner.cc:386] Master server non-default flags:
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/data
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/wal
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:31.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.124:38727
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38673/jwks.json
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/data/info.pb
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.124
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/logs
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:31.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:31.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.718147 4778 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:31.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.719872 4778 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.739032 4788 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:31.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.747195 4776 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 4
19:39:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.752739 4776 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:31.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.889539 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } attempt: 7
19:39:31.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.923856 4701 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } has no permanent_uuid. Determining permanent_uuid...
19:39:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.931618 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:31.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:31.990053 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 1
19:39:31.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.995674 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:32.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.052290 4776 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 5
19:39:32.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:32.057700 4776 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:32.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.077217 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 2
19:39:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:32.082916 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:32.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.168462 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 3
19:39:32.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:32.173904 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:32.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.314419 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 4
19:39:32.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:32.321408 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:32.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.604947 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 5
19:39:32.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.605268 4776 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 6
19:39:32.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:32.611956 4701 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:32.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:32.620483 4776 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.131.124:38727: Network error: Client connection negotiation failed: client connection to 127.4.131.124:38727: connect: Connection refused (error 111)
19:39:32.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.739439 4789 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:32.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:31.739176 4791 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:32.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:32.931964 4790 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1190 milliseconds
19:39:32.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.932123 4778 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:39:32.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.933405 4778 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:32.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.935971 4778 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:32.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.937493 4778 hybrid_clock.cc:648] HybridClock initialized: now 1790624372937442 us; error 38 us; skew 500 ppm
19:39:32.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.938691 4778 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.946595 4778 webserver.cc:460] Webserver started at http://127.4.131.124:33311/ using document root <none> and password file <none>
19:39:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.947758 4778 fs_manager.cc:359] Metadata directory not provided
19:39:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.948000 4778 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:32.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.948559 4778 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:32.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.953444 4778 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/data/instance:
19:39:32.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10eb9bc49c754104b03e5b0cc835189d"
19:39:32.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:32 on dist-test-slave-htqb"
19:39:32.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.954661 4778 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/wal/instance:
19:39:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10eb9bc49c754104b03e5b0cc835189d"
19:39:32.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:32 on dist-test-slave-htqb"
19:39:32.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.962551 4778 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
19:39:32.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.968433 4803 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:32.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.969376 4778 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.969712 4778 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/wal
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10eb9bc49c754104b03e5b0cc835189d"
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:32 on dist-test-slave-htqb"
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:32.970055 4778 fs_report.cc:383] FS layout report
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/wal
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/wal
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/data/data
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:32.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:33.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.003835 4778 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:33.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.005290 4778 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:33.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.005749 4778 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.078126 4778 rpc_server.cc:307] RPC server started. Bound to: 127.4.131.124:38727
19:39:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.078229 4854 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.131.124:38727 every 8 connection(s)
19:39:33.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.081372 4778 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/data/info.pb
19:39:33.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.088223 4855 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:39:33.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.091498 4621 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 4778
19:39:33.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.091917 4621 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/master-2/wal/instance
19:39:33.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.100660 4855 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } has no permanent_uuid. Determining permanent_uuid...
19:39:33.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.133464 4855 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } has no permanent_uuid. Determining permanent_uuid...
19:39:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.149149 4855 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } has no permanent_uuid. Determining permanent_uuid...
19:39:33.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.164561 4701 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 6
19:39:33.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.166314 4855 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: Bootstrap starting.
19:39:33.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.173414 4855 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: Neither blocks nor log segments found. Creating new log.
19:39:33.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.175649 4855 log.cc:824] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: Log is configured to *not* fsync() on all Append() calls
19:39:33.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.183079 4855 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: No bootstrap required, opened a new log
19:39:33.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.187337 4701 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367: Bootstrap starting.
19:39:33.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.194494 4701 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367: Neither blocks nor log segments found. Creating new log.
19:39:33.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.196414 4701 log.cc:824] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367: Log is configured to *not* fsync() on all Append() calls
19:39:33.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.201661 4701 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367: No bootstrap required, opened a new log
19:39:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.209784 4855 raft_consensus.cc:348] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.210669 4855 raft_consensus.cc:374] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:39:33.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.210929 4855 raft_consensus.cc:729] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 10eb9bc49c754104b03e5b0cc835189d, State: Initialized, Role: FOLLOWER
19:39:33.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.212252 4855 consensus_queue.cc:260] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.215432 4867 sys_catalog.cc:455] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.216548 4867 sys_catalog.cc:458] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [sys.catalog]: This master's current role is: FOLLOWER
19:39:33.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.217221 4855 sys_catalog.cc:564] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [sys.catalog]: configured and running, proceeding with master startup.
19:39:33.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.229058 4701 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.229687 4701 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:39:33.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.229959 4701 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a4624ac66ce4728b2df68d3c391f367, State: Initialized, Role: FOLLOWER
19:39:33.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.231285 4701 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.239501 4701 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: configured and running, proceeding with master startup.
19:39:33.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.238852 4875 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.240446 4875 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: This master's current role is: FOLLOWER
19:39:33.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:33.256804 4880 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:33.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:33.257613 4880 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:33.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:33.261228 4809 tablet.cc:2369] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:39:33.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.247195 4778 master_runner.cc:186] Error getting master registration for 127.4.131.126:35933: OK, instance_id { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" instance_seqno: 1790624369514601 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:39:33.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.289893 4778 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:39:33.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:33.302958 4891 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:33.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:33.304311 4891 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:33.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.285241 4633 master_runner.cc:186] Error getting master registration for 127.4.131.125:34735: OK, instance_id { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" instance_seqno: 1790624371293753 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:39:33.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.327060 4633 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:39:33.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:33.466981 4655 tablet.cc:2369] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:39:33.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.529690 4875 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:39:33.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.530224 4875 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.532852 4875 leader_election.cc:288] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 22cca2e2f29742b19b5cd3085a2776e5 (127.4.131.125:34735), 10eb9bc49c754104b03e5b0cc835189d (127.4.131.124:38727)
19:39:33.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.537021 4830 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a4624ac66ce4728b2df68d3c391f367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "10eb9bc49c754104b03e5b0cc835189d" is_pre_election: true
19:39:33.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.537900 4830 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3a4624ac66ce4728b2df68d3c391f367 in term 0.
19:39:33.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.539729 4653 leader_election.cc:302] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 10eb9bc49c754104b03e5b0cc835189d, 3a4624ac66ce4728b2df68d3c391f367; no voters:
19:39:33.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.541023 4875 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 0 FOLLOWER]: Leader pre-election won for term 1
19:39:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.541769 4875 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
19:39:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.542505 4875 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 0 FOLLOWER]: Advancing to term 1
19:39:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.561394 4875 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.563437 4875 leader_election.cc:288] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [CANDIDATE]: Term 1 election: Requested vote from peers 22cca2e2f29742b19b5cd3085a2776e5 (127.4.131.125:34735), 10eb9bc49c754104b03e5b0cc835189d (127.4.131.124:38727)
19:39:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.569739 4830 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a4624ac66ce4728b2df68d3c391f367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "10eb9bc49c754104b03e5b0cc835189d"
19:39:33.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.570295 4830 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [term 0 FOLLOWER]: Advancing to term 1
19:39:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.581830 4830 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3a4624ac66ce4728b2df68d3c391f367 in term 1.
19:39:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.584610 4653 leader_election.cc:302] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 10eb9bc49c754104b03e5b0cc835189d, 3a4624ac66ce4728b2df68d3c391f367; no voters:
19:39:33.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.585413 4875 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 1 FOLLOWER]: Leader election won for term 1
19:39:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.588522 4875 raft_consensus.cc:686] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [term 1 LEADER]: Becoming Leader. State: Replica: 3a4624ac66ce4728b2df68d3c391f367, State: Running, Role: LEADER
19:39:33.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.589296 4875 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.598225 4893 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3a4624ac66ce4728b2df68d3c391f367. Latest consensus state: current_term: 1 leader_uuid: "3a4624ac66ce4728b2df68d3c391f367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.599314 4893 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: This master's current role is: LEADER
19:39:33.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.604864 4895 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
19:39:33.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.611913 4895 catalog_manager.cc:1484] Initializing Kudu cluster ID...
19:39:33.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.644678 4830 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [term 1 FOLLOWER]: Refusing update from remote peer 3a4624ac66ce4728b2df68d3c391f367: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:39:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.646857 4893 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:39:33.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.681030 4776 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } attempt: 7
19:39:33.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.689402 4867 sys_catalog.cc:455] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3a4624ac66ce4728b2df68d3c391f367. Latest consensus state: current_term: 1 leader_uuid: "3a4624ac66ce4728b2df68d3c391f367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.691236 4867 sys_catalog.cc:458] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [sys.catalog]: This master's current role is: FOLLOWER
19:39:33.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.695515 4896 mvcc.cc:205] Tried to move back new op lower bound from 7334397434421026816 to 7334397434234302464. Current Snapshot: MvccSnapshot[applied={T|T < 7334397434421026816}]
19:39:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.717259 4893 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3a4624ac66ce4728b2df68d3c391f367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.718020 4893 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: This master's current role is: LEADER
19:39:33.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.726051 4867 sys_catalog.cc:455] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3a4624ac66ce4728b2df68d3c391f367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.727170 4867 sys_catalog.cc:458] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d [sys.catalog]: This master's current role is: FOLLOWER
19:39:33.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.728116 4895 catalog_manager.cc:1347] Generated new cluster ID: a67ca128a8614a96a6002d47ee6b3062
19:39:33.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.728536 4895 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
19:39:33.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.738885 4776 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5: Bootstrap starting.
19:39:33.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.755915 4776 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5: Neither blocks nor log segments found. Creating new log.
19:39:33.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.771953 4776 log.cc:824] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5: Log is configured to *not* fsync() on all Append() calls
19:39:33.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.785012 4895 catalog_manager.cc:1370] Generated new certificate authority record
19:39:33.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.789194 4895 catalog_manager.cc:1504] Loading token signing keys...
19:39:33.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.794979 4776 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5: No bootstrap required, opened a new log
19:39:33.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.832860 4895 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367: Generated new TSK 0
19:39:33.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.837138 4895 catalog_manager.cc:1514] Initializing in-progress tserver states...
19:39:33.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.862865 4776 raft_consensus.cc:348] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.863788 4776 raft_consensus.cc:374] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:39:33.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.864053 4776 raft_consensus.cc:729] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 22cca2e2f29742b19b5cd3085a2776e5, State: Initialized, Role: FOLLOWER
19:39:33.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.865003 4776 consensus_queue.cc:260] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } }
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.878741 4621 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/wal
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/data
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/logs
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/data/info.pb
19:39:33.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.65:0
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.131.65
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.65
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:33.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:39:33.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.875406 4776 sys_catalog.cc:564] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [sys.catalog]: configured and running, proceeding with master startup.
19:39:33.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.875998 4750 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a4624ac66ce4728b2df68d3c391f367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22cca2e2f29742b19b5cd3085a2776e5"
19:39:33.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.876643 4751 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3a4624ac66ce4728b2df68d3c391f367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "22cca2e2f29742b19b5cd3085a2776e5" is_pre_election: true
19:39:33.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.877637 4749 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [term 0 FOLLOWER]: Advancing to term 1
19:39:33.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.891306 4749 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [term 1 FOLLOWER]: Refusing update from remote peer 3a4624ac66ce4728b2df68d3c391f367: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
19:39:33.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.874157 4904 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.892867 4904 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [sys.catalog]: This master's current role is: FOLLOWER
19:39:33.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.892822 4875 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
19:39:33.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.974270 4904 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3a4624ac66ce4728b2df68d3c391f367. Latest consensus state: current_term: 1 leader_uuid: "3a4624ac66ce4728b2df68d3c391f367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.974350 4910 sys_catalog.cc:455] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3a4624ac66ce4728b2df68d3c391f367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:33.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.975056 4910 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [sys.catalog]: This master's current role is: FOLLOWER
19:39:33.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:33.975036 4904 sys_catalog.cc:458] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5 [sys.catalog]: This master's current role is: FOLLOWER
19:39:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.032647 4922 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
19:39:34.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.263995 4880 catalog_manager.cc:1259] Loaded cluster ID: a67ca128a8614a96a6002d47ee6b3062
19:39:34.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.264554 4880 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: loading cluster ID for follower catalog manager: success
19:39:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.272099 4880 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: acquiring CA information for follower catalog manager: success
19:39:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.278095 4880 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 10eb9bc49c754104b03e5b0cc835189d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:39:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:34.357970 4905 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:34.358912 4905 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:34.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:34.359805 4905 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:34.404368 4905 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:34.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:34.405519 4905 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.131.65
19:39:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.442888 4905 tablet_server_runner.cc:78] Tablet server non-default flags:
19:39:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/data
19:39:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/wal
19:39:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.65:0
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/data/info.pb
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.65
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:34.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.131.65
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/logs
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.444480 4905 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.446477 4905 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:34.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:34.471372 4928 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:34.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.493316 4875 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3a4624ac66ce4728b2df68d3c391f367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3a4624ac66ce4728b2df68d3c391f367" member_type: VOTER last_known_addr { host: "127.4.131.126" port: 35933 } } peers { permanent_uuid: "22cca2e2f29742b19b5cd3085a2776e5" member_type: VOTER last_known_addr { host: "127.4.131.125" port: 34735 } } peers { permanent_uuid: "10eb9bc49c754104b03e5b0cc835189d" member_type: VOTER last_known_addr { host: "127.4.131.124" port: 38727 } } }
19:39:34.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:34.494077 4875 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a4624ac66ce4728b2df68d3c391f367 [sys.catalog]: This master's current role is: LEADER
19:39:35.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:35.053753 4922 catalog_manager.cc:1259] Loaded cluster ID: a67ca128a8614a96a6002d47ee6b3062
19:39:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:35.054960 4922 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5: loading cluster ID for follower catalog manager: success
19:39:35.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:35.098860 4922 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5: acquiring CA information for follower catalog manager: success
19:39:35.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:35.143002 4922 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 22cca2e2f29742b19b5cd3085a2776e5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
19:39:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:35.862250 4927 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 4905
19:39:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:36.849129 4905 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.370s user 0.508s sys 1.085s
19:39:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:36.850729 4905 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.372s user 0.509s sys 1.085s
19:39:36.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:34.481269 4929 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:36.851799 4930 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2375 milliseconds
19:39:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.852247 4905 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:39:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:36.852252 4933 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:36.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.855875 4905 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:36.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.858297 4905 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.859758 4905 hybrid_clock.cc:648] HybridClock initialized: now 1790624376859710 us; error 41 us; skew 500 ppm
19:39:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.860667 4905 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:36.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.873294 4905 webserver.cc:460] Webserver started at http://127.4.131.65:41703/ using document root <none> and password file <none>
19:39:36.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.874387 4905 fs_manager.cc:359] Metadata directory not provided
19:39:36.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.874624 4905 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:36.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.875092 4905 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.881950 4905 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/data/instance:
19:39:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "468039ce0c5a466aab39a7316891b830"
19:39:36.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:36 on dist-test-slave-htqb"
19:39:36.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.883320 4905 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/wal/instance:
19:39:36.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "468039ce0c5a466aab39a7316891b830"
19:39:36.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:36 on dist-test-slave-htqb"
19:39:36.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.892040 4905 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
19:39:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.899307 4941 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:36.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.900506 4905 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.900851 4905 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/wal
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "468039ce0c5a466aab39a7316891b830"
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:36 on dist-test-slave-htqb"
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.901259 4905 fs_report.cc:383] FS layout report
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/wal
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/wal
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/data/data
19:39:36.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:36.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:36.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.948700 4905 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:36.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.950423 4905 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:36.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.950920 4905 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:36.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.954035 4905 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:39:36.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.959661 4905 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:39:36.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.959897 4905 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:39:36.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.960146 4905 ts_tablet_manager.cc:610] Registered 0 tablets
19:39:36.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:36.960350 4905 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:39:37.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.134609 4905 rpc_server.cc:307] RPC server started. Bound to: 127.4.131.65:45853
19:39:37.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.134748 5054 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.131.65:45853 every 8 connection(s)
19:39:37.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.136937 4905 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/data/info.pb
19:39:37.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.142596 4621 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 4905
19:39:37.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.143035 4621 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-0/wal/instance
19:39:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.156100 4621 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/wal
19:39:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/data
19:39:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:37.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/logs
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/data/info.pb
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:37.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.66:0
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.131.66
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.66
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:37.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:39:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.213824 5055 heartbeater.cc:347] Connected to a master server at 127.4.131.124:38727
19:39:37.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.214264 5055 heartbeater.cc:464] Registering TS with master...
19:39:37.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.215405 5055 heartbeater.cc:511] Master 127.4.131.124:38727 requested a full tablet report, sending...
19:39:37.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.218521 4820 ts_manager.cc:194] Registered new tserver with Master: 468039ce0c5a466aab39a7316891b830 (127.4.131.65:45853)
19:39:37.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.221872 5056 heartbeater.cc:347] Connected to a master server at 127.4.131.125:34735
19:39:37.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.222195 5056 heartbeater.cc:464] Registering TS with master...
19:39:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.222930 5057 heartbeater.cc:347] Connected to a master server at 127.4.131.126:35933
19:39:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.222971 5056 heartbeater.cc:511] Master 127.4.131.125:34735 requested a full tablet report, sending...
19:39:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.223331 5057 heartbeater.cc:464] Registering TS with master...
19:39:37.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.224093 5057 heartbeater.cc:511] Master 127.4.131.126:35933 requested a full tablet report, sending...
19:39:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.227406 4666 ts_manager.cc:194] Registered new tserver with Master: 468039ce0c5a466aab39a7316891b830 (127.4.131.65:45853)
19:39:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.227896 4741 ts_manager.cc:194] Registered new tserver with Master: 468039ce0c5a466aab39a7316891b830 (127.4.131.65:45853)
19:39:37.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.230288 4666 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.131.65:52319
19:39:37.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:37.515347 5060 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:37.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:37.516245 5060 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:37.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:37.516806 5060 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:37.560892 5060 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:37.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:37.561764 5060 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.131.66
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.596513 5060 tablet_server_runner.cc:78] Tablet server non-default flags:
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/data
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/wal
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.66:0
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/data/info.pb
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.66
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.131.66
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/logs
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.598095 5060 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:37.600169 5060 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:37.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:37.627979 5072 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:38.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.235726 5057 heartbeater.cc:503] Master 127.4.131.126:35933 was elected leader, sending a full tablet report...
19:39:38.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:37.628019 5073 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:38.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:37.636854 5075 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:38.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:38.885407 5074 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1255 milliseconds
19:39:38.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.885501 5060 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:39:38.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.886690 5060 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:38.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.889482 5060 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:38.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.890889 5060 hybrid_clock.cc:648] HybridClock initialized: now 1790624378890855 us; error 56 us; skew 500 ppm
19:39:38.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.891705 5060 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.898181 5060 webserver.cc:460] Webserver started at http://127.4.131.66:36333/ using document root <none> and password file <none>
19:39:38.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.898974 5060 fs_manager.cc:359] Metadata directory not provided
19:39:38.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.899200 5060 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:38.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.899709 5060 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:38.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.904091 5060 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/data/instance:
19:39:38.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39c93611c8d146ab9e478aa9a19a1a79"
19:39:38.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:38 on dist-test-slave-htqb"
19:39:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.905076 5060 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/wal/instance:
19:39:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39c93611c8d146ab9e478aa9a19a1a79"
19:39:38.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:38 on dist-test-slave-htqb"
19:39:38.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.911528 5060 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.005s
19:39:38.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.916575 5083 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:38.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.917627 5060 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
19:39:38.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.917969 5060 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/wal
19:39:38.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "39c93611c8d146ab9e478aa9a19a1a79"
19:39:38.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:38 on dist-test-slave-htqb"
19:39:38.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.918301 5060 fs_report.cc:383] FS layout report
19:39:38.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/wal
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/wal
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/data/data
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:38.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.949947 5060 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.951355 5060 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:38.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.951854 5060 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:38.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.954262 5060 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:39:38.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.958071 5060 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:39:38.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.958295 5060 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:39:38.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.958640 5060 ts_tablet_manager.cc:610] Registered 0 tablets
19:39:38.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:38.958843 5060 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:39:39.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.104982 5060 rpc_server.cc:307] RPC server started. Bound to: 127.4.131.66:37135
19:39:39.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.105043 5195 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.131.66:37135 every 8 connection(s)
19:39:39.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.108215 5060 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/data/info.pb
19:39:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.116333 4621 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 5060
19:39:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.116997 4621 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-1/wal/instance
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.126926 4621 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/wal
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/data
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:39.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/logs
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/data/info.pb
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.67:0
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.131.67
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.67
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:39.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
19:39:39.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.182696 5196 heartbeater.cc:347] Connected to a master server at 127.4.131.124:38727
19:39:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.183118 5196 heartbeater.cc:464] Registering TS with master...
19:39:39.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.184870 5196 heartbeater.cc:511] Master 127.4.131.124:38727 requested a full tablet report, sending...
19:39:39.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.188067 5197 heartbeater.cc:347] Connected to a master server at 127.4.131.125:34735
19:39:39.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.188175 4820 ts_manager.cc:194] Registered new tserver with Master: 39c93611c8d146ab9e478aa9a19a1a79 (127.4.131.66:37135)
19:39:39.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.188410 5197 heartbeater.cc:464] Registering TS with master...
19:39:39.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.189555 5197 heartbeater.cc:511] Master 127.4.131.125:34735 requested a full tablet report, sending...
19:39:39.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.192106 4741 ts_manager.cc:194] Registered new tserver with Master: 39c93611c8d146ab9e478aa9a19a1a79 (127.4.131.66:37135)
19:39:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.197158 5198 heartbeater.cc:347] Connected to a master server at 127.4.131.126:35933
19:39:39.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.197528 5198 heartbeater.cc:464] Registering TS with master...
19:39:39.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.198289 5198 heartbeater.cc:511] Master 127.4.131.126:35933 requested a full tablet report, sending...
19:39:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.200281 4666 ts_manager.cc:194] Registered new tserver with Master: 39c93611c8d146ab9e478aa9a19a1a79 (127.4.131.66:37135)
19:39:39.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.201565 4666 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.131.66:41071
19:39:39.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:39.490527 5202 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:39.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:39.491089 5202 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:39.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:39.491614 5202 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:39.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:39.520851 5202 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:39.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:39.521678 5202 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.4.131.67
19:39:39.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.555593 5202 tablet_server_runner.cc:78] Tablet server non-default flags:
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.131.84:37047
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/data
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/wal
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.131.67:0
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/data/info.pb
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.131.67
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.4.131.126:35933,127.4.131.125:34735,127.4.131.124:38727
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.4.131.67
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/logs
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:39.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:39.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.556808 5202 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:39.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.558394 5202 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:39.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:39.578433 5213 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:40.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.205421 5198 heartbeater.cc:503] Master 127.4.131.126:35933 was elected leader, sending a full tablet report...
19:39:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:39.578769 5212 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:39.581056 5215 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:39.580243 5202 server_base.cc:1034] running on GCE node
19:39:40.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.729327 5202 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:40.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.731429 5202 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:40.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.732818 5202 hybrid_clock.cc:648] HybridClock initialized: now 1790624380732784 us; error 55 us; skew 500 ppm
19:39:40.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.733631 5202 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:40.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.739733 5202 webserver.cc:460] Webserver started at http://127.4.131.67:38849/ using document root <none> and password file <none>
19:39:40.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.740576 5202 fs_manager.cc:359] Metadata directory not provided
19:39:40.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.740823 5202 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:40.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.741255 5202 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.745703 5202 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/data/instance:
19:39:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3178d2099a444aca55acc14e65b9651"
19:39:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:40 on dist-test-slave-htqb"
19:39:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.746644 5202 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/wal/instance:
19:39:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3178d2099a444aca55acc14e65b9651"
19:39:40.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:40 on dist-test-slave-htqb"
19:39:40.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.753041 5202 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.003s sys 0.006s
19:39:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.758046 5224 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.759191 5202 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
19:39:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.759451 5202 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/wal
19:39:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a3178d2099a444aca55acc14e65b9651"
19:39:40.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:40 on dist-test-slave-htqb"
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.759716 5202 fs_report.cc:383] FS layout report
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/wal
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/wal
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/data/data
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:40.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.813808 5202 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:40.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.815070 5202 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:40.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.815485 5202 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:40.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.817806 5202 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
19:39:40.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.821831 5202 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
19:39:40.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.821998 5202 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
19:39:40.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.822181 5202 ts_tablet_manager.cc:610] Registered 0 tablets
19:39:40.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.822300 5202 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
19:39:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.969919 5202 rpc_server.cc:307] RPC server started. Bound to: 127.4.131.67:38541
19:39:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.970000 5336 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.131.67:38541 every 8 connection(s)
19:39:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.972108 5202 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/data/info.pb
19:39:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.975406 4621 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 5202
19:39:40.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:40.975850 4621 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster2121876445486826498/ts-2/wal/instance
19:39:41.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.011361 5337 heartbeater.cc:347] Connected to a master server at 127.4.131.124:38727
19:39:41.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.011866 5337 heartbeater.cc:464] Registering TS with master...
19:39:41.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.013505 5337 heartbeater.cc:511] Master 127.4.131.124:38727 requested a full tablet report, sending...
19:39:41.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.016080 5338 heartbeater.cc:347] Connected to a master server at 127.4.131.125:34735
19:39:41.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.017037 5338 heartbeater.cc:464] Registering TS with master...
19:39:41.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.017517 4820 ts_manager.cc:194] Registered new tserver with Master: a3178d2099a444aca55acc14e65b9651 (127.4.131.67:38541)
19:39:41.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.017895 5338 heartbeater.cc:511] Master 127.4.131.125:34735 requested a full tablet report, sending...
19:39:41.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.020231 4741 ts_manager.cc:194] Registered new tserver with Master: a3178d2099a444aca55acc14e65b9651 (127.4.131.67:38541)
19:39:41.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.025704 5340 heartbeater.cc:347] Connected to a master server at 127.4.131.126:35933
19:39:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.026024 5340 heartbeater.cc:464] Registering TS with master...
19:39:41.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.026715 5340 heartbeater.cc:511] Master 127.4.131.126:35933 requested a full tablet report, sending...
19:39:41.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.028892 4666 ts_manager.cc:194] Registered new tserver with Master: a3178d2099a444aca55acc14e65b9651 (127.4.131.67:38541)
19:39:41.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.030755 4666 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.4.131.67:48959
19:39:41.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.036281 4621 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
19:39:41.045 [DEBUG - main] (MiniKuduCluster.java:195) Response:
19:39:41.046 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
19:39:41.057 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.131.126"
port: 35933
}
bound_http_address {
host: "127.4.131.126"
port: 33141
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.131.125"
port: 34735
}
bound_http_address {
host: "127.4.131.125"
port: 42645
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.131.124"
port: 38727
}
bound_http_address {
host: "127.4.131.124"
port: 33311
}
}
}
19:39:41.063 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:39:41.066 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.4.131.65"
port: 45853
}
bound_http_address {
host: "127.4.131.65"
port: 41703
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.4.131.66"
port: 37135
}
bound_http_address {
host: "127.4.131.66"
port: 36333
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.4.131.67"
port: 38541
}
bound_http_address {
host: "127.4.131.67"
port: 38849
}
}
}
19:39:41.066 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
19:39:41.234 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
19:39:41.353 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task9OVAZw/test-tmp
19:39:41.353 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
19:39:41.354 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task9OVAZw/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
19:39:41.359 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
19:39:41.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:41.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
19:39:41.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.697993 5348 mini_oidc.cc:152] Starting JWKS server
19:39:41.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.701632 5348 webserver.cc:460] Webserver started at http://127.0.0.1:34345/ using document root <none> and password file <none>
19:39:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.702111 5348 mini_oidc.cc:189] Starting OIDC Discovery server
19:39:41.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:41.703826 5348 webserver.cc:460] Webserver started at http://127.0.0.1:35413/ using document root <none> and password file <none>
19:39:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
19:39:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/principal' for realm 'KRBTEST.COM',
19:39:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
19:39:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:41 dist-test-slave-htqb krb5kdc[5361](info): setting up network...
19:39:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
19:39:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:41 dist-test-slave-htqb krb5kdc[5361](info): set up 2 sockets
19:39:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:41 dist-test-slave-htqb krb5kdc[5361](info): commencing operation
19:39:41.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
19:39:42.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:42.034509 5340 heartbeater.cc:503] Master 127.4.131.126:35933 was elected leader, sending a full tablet report...
19:39:42.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:42.785619 5348 mini_kdc.cc:121] Time spent starting KDC: real 1.059s user 0.006s sys 0.013s
19:39:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
19:39:42.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:39:42.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
19:39:42.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
19:39:42.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:39:42.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
19:39:42.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
19:39:42.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:39:42.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
19:39:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
19:39:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/test-user.keytab.
19:39:42.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/test-user.keytab.
19:39:42.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:42 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624382, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:42.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:39:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:39:42Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
19:39:42.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:39:42Z Disabled control of system clock
19:39:42.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.57.62@KRBTEST.COM; defaulting to no policy
19:39:42.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:39:42.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.57.62@KRBTEST.COM" created.
19:39:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:39:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.57.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.62.keytab.
19:39:42.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.57.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.62.keytab.
19:39:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:42.971486 5348 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:42.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/logs
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/info.pb
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.62:39269
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.62
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.62.keytab
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.62
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:42.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:39:43.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.282583 5374 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:43.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.283159 5374 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:43.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.284019 5374 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:43.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.315227 5374 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:39:43.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.315440 5374 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:43.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.315697 5374 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.315886 5374 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.316107 5374 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.316352 5374 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:43.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.316494 5374 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34345/jwks.json
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:43.353794 5374 master_runner.cc:386] Master server non-default flags:
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.62.keytab
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.62
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.62:39269
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:43.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/info.pb
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.62
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/logs
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:43.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:43.354939 5374 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:43.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:43.355911 5374 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:43.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.375612 5383 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:44.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.375607 5380 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:44.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:43.375876 5381 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:44.637144 5382 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1259 milliseconds
19:39:44.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.637259 5374 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:39:44.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.638412 5374 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:44.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.640952 5374 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:44.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.642344 5374 hybrid_clock.cc:648] HybridClock initialized: now 1790624384642323 us; error 59 us; skew 500 ppm
19:39:44.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.643141 5374 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:44.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:44 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624384, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:44.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.660668 5374 init.cc:377] Logged in from keytab as kudu/127.5.57.62@KRBTEST.COM (short username kudu)
19:39:44.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.666532 5374 webserver.cc:460] Webserver started at http://127.5.57.62:45793/ using document root <none> and password file <none>
19:39:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.667531 5374 fs_manager.cc:359] Metadata directory not provided
19:39:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.667737 5374 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:44.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.668128 5374 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.672776 5374 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/instance:
19:39:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9179aae1af1247599294b6b041d1d97a"
19:39:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:44 on dist-test-slave-htqb"
19:39:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.673825 5374 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal/instance:
19:39:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9179aae1af1247599294b6b041d1d97a"
19:39:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:44 on dist-test-slave-htqb"
19:39:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.681329 5374 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
19:39:44.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.686601 5392 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:44.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.687717 5374 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
19:39:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.688086 5374 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9179aae1af1247599294b6b041d1d97a"
19:39:44.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:44 on dist-test-slave-htqb"
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.688446 5374 fs_report.cc:383] FS layout report
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/data
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:44.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.734920 5374 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:44.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.740603 5374 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:44.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.741117 5374 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.819242 5374 rpc_server.cc:307] RPC server started. Bound to: 127.5.57.62:39269
19:39:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.819309 5443 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.57.62:39269 every 8 connection(s)
19:39:44.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.822234 5374 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/info.pb
19:39:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.827831 5444 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:39:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.829963 5348 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 5374
19:39:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.830430 5348 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal/instance
19:39:44.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.839568 5444 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } has no permanent_uuid. Determining permanent_uuid...
19:39:44.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.57.61@KRBTEST.COM; defaulting to no policy
19:39:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:39:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.57.61@KRBTEST.COM" created.
19:39:44.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:44 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624384, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.62@KRBTEST.COM for kudu/127.5.57.62@KRBTEST.COM
19:39:44.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:39:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.57.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.61.keytab.
19:39:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.57.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.61.keytab.
19:39:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.879771 5348 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data
19:39:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/logs
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/info.pb
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.61:42765
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.61
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.61.keytab
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.61
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:39:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.883041 5444 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } has no permanent_uuid. Determining permanent_uuid...
19:39:44.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:44.885458 5394 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:44.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:44.889777 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.61:42765: Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:44.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:44.939195 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } attempt: 1
19:39:44.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:44.942710 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.61:42765: Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:45.011065 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } attempt: 2
19:39:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.014390 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.61:42765: Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:45.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:45.105820 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } attempt: 3
19:39:45.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.109942 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.61:42765: Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:45.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.212773 5450 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:45.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.213434 5450 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:45.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.214282 5450 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.247738 5450 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:39:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.248162 5450 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:45.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.248652 5450 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:45.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.249140 5450 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:45.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.249652 5450 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:45.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.249985 5450 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:45.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.250240 5450 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34345/jwks.json
19:39:45.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:45.253324 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } attempt: 4
19:39:45.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.256647 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.61:42765: Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:45.290696 5450 master_runner.cc:386] Master server non-default flags:
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.61.keytab
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.61
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.61:42765
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/info.pb
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.61
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/logs
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:45.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:45.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:45.291868 5450 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:45.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:45.292944 5450 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:45.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.313380 5460 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:45.555997 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } attempt: 5
19:39:45.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.562664 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.61:42765: Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:46.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.110198 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } attempt: 6
19:39:46.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:46.160270 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.61:42765: Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:46.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.313874 5458 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:46.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:45.315786 5457 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:46.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:46.638317 5459 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1322 milliseconds
19:39:46.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.638403 5450 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
19:39:46.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.639727 5450 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.645531 5450 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.646932 5450 hybrid_clock.cc:648] HybridClock initialized: now 1790624386646906 us; error 41 us; skew 500 ppm
19:39:46.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.647868 5450 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:46.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:46 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624386, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:46.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.654016 5450 init.cc:377] Logged in from keytab as kudu/127.5.57.61@KRBTEST.COM (short username kudu)
19:39:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.659688 5450 webserver.cc:460] Webserver started at http://127.5.57.61:46191/ using document root <none> and password file <none>
19:39:46.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.660676 5450 fs_manager.cc:359] Metadata directory not provided
19:39:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.660920 5450 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.661415 5450 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:46.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.666183 5450 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/instance:
19:39:46.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6994105f778e464f95640f88841736cc"
19:39:46.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:46 on dist-test-slave-htqb"
19:39:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.667410 5450 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal/instance:
19:39:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6994105f778e464f95640f88841736cc"
19:39:46.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:46 on dist-test-slave-htqb"
19:39:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.674959 5450 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
19:39:46.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.680687 5470 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:46.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.681672 5450 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
19:39:46.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.682029 5450 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:46.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6994105f778e464f95640f88841736cc"
19:39:46.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:46 on dist-test-slave-htqb"
19:39:46.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.682389 5450 fs_report.cc:383] FS layout report
19:39:46.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/data
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:46.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:46.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:46.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:46.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.717478 5450 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:46.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.723604 5450 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:46.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.724118 5450 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.802132 5450 rpc_server.cc:307] RPC server started. Bound to: 127.5.57.61:42765
19:39:46.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.802212 5521 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.57.61:42765 every 8 connection(s)
19:39:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.804962 5450 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/info.pb
19:39:46.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.810665 5522 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:39:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.810928 5348 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 5450
19:39:46.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.811374 5348 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal/instance
19:39:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.5.57.60@KRBTEST.COM; defaulting to no policy
19:39:46.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.825875 5522 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } has no permanent_uuid. Determining permanent_uuid...
19:39:46.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:39:46.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.5.57.60@KRBTEST.COM" created.
19:39:46.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:46 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624386, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.61@KRBTEST.COM for kudu/127.5.57.62@KRBTEST.COM
19:39:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
19:39:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.57.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.60.keytab.
19:39:46.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.5.57.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.60.keytab.
19:39:46.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.859565 5348 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/logs
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/info.pb
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.60:40783
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.60
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.60.keytab
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.60
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:46.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:46.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:39:46.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.868420 5522 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } has no permanent_uuid. Determining permanent_uuid...
19:39:46.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:46 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624386, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.61@KRBTEST.COM for kudu/127.5.57.61@KRBTEST.COM
19:39:46.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.894833 5522 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } has no permanent_uuid. Determining permanent_uuid...
19:39:46.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:46.896893 5474 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:46.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:46.901113 5522 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:46.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:46.950560 5522 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 1
19:39:46.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:46.954442 5522 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.022792 5522 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 2
19:39:47.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.027475 5522 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.118855 5522 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 3
19:39:47.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.122979 5522 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.203637 5528 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:47.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.204165 5528 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:47.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.204797 5528 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:47.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.220860 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } attempt: 7
19:39:47.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:47 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624384, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.62@KRBTEST.COM for kudu/127.5.57.61@KRBTEST.COM
19:39:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.246001 5528 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:39:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.246263 5528 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.246688 5528 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:47.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.246654 5444 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } has no permanent_uuid. Determining permanent_uuid...
19:39:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.247044 5528 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.247424 5528 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:47.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.247766 5528 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:47.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.248023 5528 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34345/jwks.json
19:39:47.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.251559 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.266456 5522 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 4
19:39:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.270787 5522 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.288311 5528 master_runner.cc:386] Master server non-default flags:
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.60.keytab
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.60
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.60:40783
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/info.pb
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.60
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/logs
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:47.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:47.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:47.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:47.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:47.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:47.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.289620 5528 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:47.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.290560 5528 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.309813 5539 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:47.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.309912 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 1
19:39:47.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.318486 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.399885 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 2
19:39:47.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.403739 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.489342 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 3
19:39:47.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.494861 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.570218 5522 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 5
19:39:47.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.574755 5522 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.635564 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 4
19:39:47.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.640434 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:47.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.923864 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 5
19:39:47.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.928551 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:48.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.122177 5522 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 6
19:39:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:48.129577 5522 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:48.171316 4697 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5740 after lost signal to thread 4634
19:39:48.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:48.173270 4697 debug-util.cc:398] Leaking SignalData structure 0x7b0800083e80 after lost signal to thread 4700
19:39:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.480979 5444 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } attempt: 6
19:39:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:48.487977 5444 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.5.57.60:40783: Network error: Client connection negotiation failed: client connection to 127.5.57.60:40783: connect: Connection refused (error 111)
19:39:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.313557 5541 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:47.312538 5528 server_base.cc:1034] running on GCE node
19:39:48.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:47.311645 5538 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:48.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.642769 5528 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:48.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.646037 5528 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:48.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.647469 5528 hybrid_clock.cc:648] HybridClock initialized: now 1790624388647439 us; error 52 us; skew 500 ppm
19:39:48.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.648278 5528 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:48 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624388, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:48.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.654712 5528 init.cc:377] Logged in from keytab as kudu/127.5.57.60@KRBTEST.COM (short username kudu)
19:39:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.660667 5528 webserver.cc:460] Webserver started at http://127.5.57.60:40025/ using document root <none> and password file <none>
19:39:48.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.661562 5528 fs_manager.cc:359] Metadata directory not provided
19:39:48.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.661798 5528 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:48.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.662303 5528 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
19:39:48.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.667021 5528 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/instance:
19:39:48.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc50be80daea41909bbe4a0f857f0ae7"
19:39:48.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:48 on dist-test-slave-htqb"
19:39:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.668186 5528 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal/instance:
19:39:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc50be80daea41909bbe4a0f857f0ae7"
19:39:48.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:48 on dist-test-slave-htqb"
19:39:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.676344 5528 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.003s sys 0.005s
19:39:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.683635 5556 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.684813 5528 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
19:39:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.685216 5528 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc50be80daea41909bbe4a0f857f0ae7"
19:39:48.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:48 on dist-test-slave-htqb"
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.685647 5528 fs_report.cc:383] FS layout report
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/data
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:48.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:48.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:48.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:48.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:48.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:48.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.728430 5528 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:48.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.734429 5528 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:48.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.734907 5528 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:48.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.833961 5528 rpc_server.cc:307] RPC server started. Bound to: 127.5.57.60:40783
19:39:48.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.834059 5607 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.57.60:40783 every 8 connection(s)
19:39:48.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.836787 5528 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/info.pb
19:39:48.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.838681 5348 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 5528
19:39:48.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.839285 5348 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal/instance
19:39:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.844281 5608 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
19:39:48.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.862426 5608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } has no permanent_uuid. Determining permanent_uuid...
19:39:48.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:48 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624388, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.60@KRBTEST.COM for kudu/127.5.57.62@KRBTEST.COM
19:39:48.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:48 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624382, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.57.60@KRBTEST.COM
19:39:48.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.906440 5608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } has no permanent_uuid. Determining permanent_uuid...
19:39:48.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:48 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624388, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.60@KRBTEST.COM for kudu/127.5.57.61@KRBTEST.COM
19:39:48.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:48 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624382, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.57.61@KRBTEST.COM
19:39:48.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.936097 5608 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } has no permanent_uuid. Determining permanent_uuid...
19:39:48.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:48 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624388, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.60@KRBTEST.COM for kudu/127.5.57.60@KRBTEST.COM
19:39:48.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:48 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624382, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.57.62@KRBTEST.COM
19:39:48.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.972043 5608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: Bootstrap starting.
19:39:48.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.975467 5348 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
19:39:48.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.979504 5608 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: Neither blocks nor log segments found. Creating new log.
19:39:48.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.982071 5608 log.cc:824] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: Log is configured to *not* fsync() on all Append() calls
19:39:48.988 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:48.989 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
19:39:48.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:48.991773 5608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: No bootstrap required, opened a new log
19:39:49.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
19:39:49.003 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
19:39:49.006 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.5.57.62"
port: 39269
}
bound_http_address {
host: "127.5.57.62"
port: 45793
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.5.57.61"
port: 42765
}
bound_http_address {
host: "127.5.57.61"
port: 46191
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.5.57.60"
port: 40783
}
bound_http_address {
host: "127.5.57.60"
port: 40025
}
}
}
19:39:49.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
19:39:49.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:49.008 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:39:49.010 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:39:49.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.015502 5608 raft_consensus.cc:348] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:39:49.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.016294 5608 raft_consensus.cc:374] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:39:49.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.016563 5608 raft_consensus.cc:729] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cc50be80daea41909bbe4a0f857f0ae7, State: Initialized, Role: FOLLOWER
19:39:49.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.017500 5608 consensus_queue.cc:260] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:39:49.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.020926 5618 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } } }
19:39:49.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.021843 5618 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [sys.catalog]: This master's current role is: FOLLOWER
19:39:49.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.022743 5608 sys_catalog.cc:564] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [sys.catalog]: configured and running, proceeding with master startup.
19:39:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:49 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624389, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:49.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:39:49.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.057868 5630 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:49.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.058172 5630 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:49.062 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:49.064 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc
19:39:49.073 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:39:49.074 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc
19:39:49.077 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:39:49.081 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
19:39:49.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.059351 5528 master_runner.cc:186] Error getting master registration for 127.5.57.62:39269: OK, instance_id { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" instance_seqno: 1790624384798055 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
19:39:49.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.093842 5528 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:39:49.094 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
19:39:49.094 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
19:39:49.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.129000 5618 raft_consensus.cc:482] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:39:49.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.129562 5618 raft_consensus.cc:504] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:39:49.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:49 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624389, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:49.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
19:39:49.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:49.145 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc
19:39:49.146 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
19:39:49.148 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.57.61:42765
19:39:49.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.144963 5618 leader_election.cc:288] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269), 6994105f778e464f95640f88841736cc (127.5.57.61:42765)
19:39:49.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
19:39:49.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.150554 5348 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 5450
19:39:49.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.182017 5558 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.5.57.61:42765 (error 108)
19:39:49.187 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:49.188 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.57.60:40783
19:39:49.188 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
19:39:49.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.188719 5348 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 5528
19:39:49.218 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:49.218 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.5.57.62:39269
19:39:49.218 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
19:39:49.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.218892 5348 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 5374
19:39:49.244 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:49.244 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.57.61:42765
19:39:49.246 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
19:39:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.247434 5348 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:49.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/logs
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/info.pb
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.61:42765
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.61
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46191
19:39:49.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.61.keytab
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.61
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:49.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:39:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.585781 5635 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.586274 5635 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:49.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.586797 5635 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:49.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.620702 5635 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:39:49.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.620911 5635 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:49.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.621168 5635 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:49.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.621343 5635 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:49.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.621510 5635 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:49.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.621672 5635 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:49.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.621791 5635 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34345/jwks.json
19:39:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.660118 5635 master_runner.cc:386] Master server non-default flags:
19:39:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data
19:39:49.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.61.keytab
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.61
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.61:42765
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/info.pb
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.61
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46191
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/logs
19:39:49.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.661325 5635 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:49.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.662189 5635 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:49.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.680821 5643 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:50.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.681936 5642 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:50.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:49.683099 5635 server_base.cc:1034] running on GCE node
19:39:50.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:49.682633 5645 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:50.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.915787 5635 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:50.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.918972 5635 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:50.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.920406 5635 hybrid_clock.cc:648] HybridClock initialized: now 1790624390920362 us; error 66 us; skew 500 ppm
19:39:50.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.921244 5635 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:50.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:50 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624390, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:50.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.933398 5635 init.cc:377] Logged in from keytab as kudu/127.5.57.61@KRBTEST.COM (short username kudu)
19:39:50.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.938479 5635 webserver.cc:460] Webserver started at http://127.5.57.61:46191/ using document root <none> and password file <none>
19:39:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.939438 5635 fs_manager.cc:359] Metadata directory not provided
19:39:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.939634 5635 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:50.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.946853 5635 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.006s sys 0.001s
19:39:50.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.950793 5654 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:50.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.951857 5635 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.952121 5635 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6994105f778e464f95640f88841736cc"
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:46 on dist-test-slave-htqb"
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.954398 5635 fs_report.cc:383] FS layout report
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/data
19:39:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:50.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:50.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:50.997574 5635 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:51.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.002543 5635 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:51.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.002942 5635 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.073559 5635 rpc_server.cc:307] RPC server started. Bound to: 127.5.57.61:42765
19:39:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.073711 5705 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.57.61:42765 every 8 connection(s)
19:39:51.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.076694 5635 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/data/info.pb
19:39:51.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.083771 5348 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 5635
19:39:51.084 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:51.084 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.57.60:40783
19:39:51.084 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
19:39:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.086313 5348 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data
19:39:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:51.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/logs
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/info.pb
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:51.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.60:40783
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.60
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40025
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.60.keytab
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.60
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:51.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:39:51.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.093228 5706 sys_catalog.cc:263] Verifying existing consensus state
19:39:51.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 19:39:51.094008 5706 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:39:51.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.094432 5635 master.cc:537] Master@127.5.57.61:42765 shutting down...
19:39:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.153250 5635 master.cc:552] Master@127.5.57.61:42765 shutdown complete.
19:39:51.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:39:51.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.404323 5708 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:51.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.405004 5708 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:51.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.405730 5708 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:51.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.438016 5708 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:39:51.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.438309 5708 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:51.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.438730 5708 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:51.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.439090 5708 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:51.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.439435 5708 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.439757 5708 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.440033 5708 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34345/jwks.json
19:39:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.477097 5708 master_runner.cc:386] Master server non-default flags:
19:39:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data
19:39:51.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.60.keytab
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.60
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.60:40783
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:51.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/info.pb
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.60
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40025
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/logs
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:51.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.478700 5708 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:51.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.480060 5708 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:51.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.499189 5714 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:52.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.499181 5715 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:52.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:51.501271 5708 server_base.cc:1034] running on GCE node
19:39:52.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:51.499931 5717 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.790786 5708 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.794451 5708 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:52.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.798159 5708 hybrid_clock.cc:648] HybridClock initialized: now 1790624392798063 us; error 127 us; skew 500 ppm
19:39:52.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.799307 5708 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:52.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:52 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624392, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:52.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.806042 5708 init.cc:377] Logged in from keytab as kudu/127.5.57.60@KRBTEST.COM (short username kudu)
19:39:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.811627 5708 webserver.cc:460] Webserver started at http://127.5.57.60:40025/ using document root <none> and password file <none>
19:39:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.812670 5708 fs_manager.cc:359] Metadata directory not provided
19:39:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.812877 5708 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.821219 5708 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
19:39:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.826469 5727 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:52.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.827548 5708 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
19:39:52.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.827908 5708 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:52.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cc50be80daea41909bbe4a0f857f0ae7"
19:39:52.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:48 on dist-test-slave-htqb"
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.830672 5708 fs_report.cc:383] FS layout report
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/wal
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/data
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:52.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:52.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.897979 5708 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:52.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.914639 5708 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.915644 5708 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:52.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.993322 5708 rpc_server.cc:307] RPC server started. Bound to: 127.5.57.60:40783
19:39:52.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.993499 5778 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.57.60:40783 every 8 connection(s)
19:39:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.995993 5708 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-2/data/info.pb
19:39:52.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:52.998342 5348 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 5708
19:39:52.999 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:53.000 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.5.57.62:39269
19:39:53.000 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.001978 5348 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/logs
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/info.pb
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:53.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.62:39269
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.62
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45793
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.62.keytab
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.62
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:53.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
19:39:53.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.009251 5779 sys_catalog.cc:263] Verifying existing consensus state
19:39:53.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.014315 5779 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: Bootstrap starting.
19:39:53.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.044142 5779 log.cc:824] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: Log is configured to *not* fsync() on all Append() calls
19:39:53.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.048178 5779 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
19:39:53.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.048728 5779 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: Bootstrap complete.
19:39:53.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.073443 5779 raft_consensus.cc:348] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:39:53.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.075963 5779 raft_consensus.cc:374] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
19:39:53.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.076233 5779 raft_consensus.cc:729] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: cc50be80daea41909bbe4a0f857f0ae7, State: Initialized, Role: FOLLOWER
19:39:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.077099 5779 consensus_queue.cc:260] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:39:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.081720 5783 sys_catalog.cc:455] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } } }
19:39:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.082494 5783 sys_catalog.cc:458] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [sys.catalog]: This master's current role is: FOLLOWER
19:39:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.083268 5779 sys_catalog.cc:564] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [sys.catalog]: configured and running, proceeding with master startup.
19:39:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.134156 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:53.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.134454 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.129817 5708 master_runner.cc:186] Error getting master registration for 127.5.57.62:39269: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111),
19:39:53.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.154613 5708 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:39:53.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.172871 5733 tablet.cc:2369] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: Can't schedule compaction. Clean time has not been advanced past its initial value.
19:39:53.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.345837 5781 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
19:39:53.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.346496 5781 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
19:39:53.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.347332 5781 flags.cc:425] Enabled unsafe flag: --never_fsync=true
19:39:53.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.379045 5781 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
19:39:53.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.379387 5781 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
19:39:53.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.379776 5781 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
19:39:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.380091 5781 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
19:39:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.380391 5781 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
19:39:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.380697 5781 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
19:39:53.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.380923 5781 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34345/jwks.json
19:39:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.420382 5781 master_runner.cc:386] Master server non-default flags:
19:39:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
19:39:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.5.57.20:42257
19:39:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
19:39:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
19:39:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data
19:39:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
19:39:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
19:39:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.5.57.62:39269,127.5.57.61:42765,127.5.57.60:40783
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/krb5kdc/kudu_127.5.57.62.keytab
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.5.57.62
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.5.57.62:39269
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34345/jwks.json
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/info.pb
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.5.57.62
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45793
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
19:39:53.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/logs
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 28 Sep 2026 18:40:50 UTC on ccycloud.quasar-innkwk.root.comops.site
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4677987
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.421823 5781 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
19:39:53.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.423205 5781 file_cache.cc:493] Constructed file cache file cache with capacity 419430
19:39:53.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.442466 5805 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:53.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.560923 5783 raft_consensus.cc:482] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:39:53.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.561687 5783 raft_consensus.cc:504] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:39:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.567955 5731 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111)
19:39:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.568303 5783 leader_election.cc:288] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269), 6994105f778e464f95640f88841736cc (127.5.57.61:42765)
19:39:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.581135 5731 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269): Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111)
19:39:53.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.583499 5729 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6994105f778e464f95640f88841736cc (127.5.57.61:42765): Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:53.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.584080 5729 leader_election.cc:302] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cc50be80daea41909bbe4a0f857f0ae7; no voters: 6994105f778e464f95640f88841736cc, 9179aae1af1247599294b6b041d1d97a
19:39:53.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.585178 5783 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:39:54.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:54.138960 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:54.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:54.139627 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:54.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.443830 5802 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:53.444135 5803 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
19:39:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:53.444618 5781 server_base.cc:1034] running on GCE node
19:39:54.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.750614 5781 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
19:39:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.754315 5781 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
19:39:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.755752 5781 hybrid_clock.cc:648] HybridClock initialized: now 1790624394755679 us; error 76 us; skew 500 ppm
19:39:54.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.756517 5781 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
19:39:54.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:54 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624394, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:39:54.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.764271 5781 init.cc:377] Logged in from keytab as kudu/127.5.57.62@KRBTEST.COM (short username kudu)
19:39:54.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.770359 5781 webserver.cc:460] Webserver started at http://127.5.57.62:45793/ using document root <none> and password file <none>
19:39:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.771478 5781 fs_manager.cc:359] Metadata directory not provided
19:39:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.771742 5781 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
19:39:54.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.780637 5781 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.003s
19:39:54.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.785838 5816 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
19:39:54.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.787068 5781 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
19:39:54.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.787514 5781 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data,/tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9179aae1af1247599294b6b041d1d97a"
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-28 19:39:44 on dist-test-slave-htqb"
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.790918 5781 fs_report.cc:383] FS layout report
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/data
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
19:39:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
19:39:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
19:39:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
19:39:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
19:39:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
19:39:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
19:39:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
19:39:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
19:39:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
19:39:54.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.833314 5781 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
19:39:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.840463 5781 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
19:39:54.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.841082 5781 kserver.cc:163] Server-wide thread pool size limit: 3276
19:39:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.962476 5781 rpc_server.cc:307] RPC server started. Bound to: 127.5.57.62:39269
19:39:54.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.962661 5867 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.5.57.62:39269 every 8 connection(s)
19:39:54.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.965838 5781 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/data/info.pb
19:39:54.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
19:39:54.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.967869 5348 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu as pid 5781
19:39:54.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.990494 5868 sys_catalog.cc:263] Verifying existing consensus state
19:39:54.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260928 19:39:54.991456 5868 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:39:54.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:54.991874 5781 master.cc:537] Master@127.5.57.62:39269 shutting down...
19:39:55.048 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:55.090524 5781 master.cc:552] Master@127.5.57.62:39269 shutdown complete.
19:39:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9OVAZw/test-tmp/mini-kudu-cluster3428600299159628850/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
19:39:55.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:55.146162 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:55.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:55.146731 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:55.159 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.57.60:40783(127.5.57.60:40783)] connecting to peer
19:39:55.160 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.213 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.5.57.60:40783(127.5.57.60:40783)] Successfully connected to peer
19:39:55.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.220 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.303 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
19:39:55.313 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.5.57.60
19:39:55.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:39:55 dist-test-slave-htqb krb5kdc[5361](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624389, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.5.57.60@KRBTEST.COM
19:39:55.551 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x94ce4587, L:/127.0.0.1:55730 - R:/127.5.57.60:40783] using SASL/GSSAPI
19:39:55.566 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:55.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.583 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.585 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.586 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.586 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.591 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:55.599 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.602 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.604 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.604 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.606 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.607 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.607 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.608 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.608 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:55.619 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.620 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.622 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.623 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.623 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.624 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.624 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.628 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:55.659 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.661 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.663 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.663 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.664 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.665 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.665 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.667 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:55.699 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.701 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.703 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.703 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.704 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.704 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.705 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.705 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.707 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:55.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:55.718310 5874 raft_consensus.cc:482] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:39:55.720 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:55.719673 5874 raft_consensus.cc:504] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:39:55.723 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:55.723644 5874 leader_election.cc:288] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269), 6994105f778e464f95640f88841736cc (127.5.57.61:42765)
19:39:55.727 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.727 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.728 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:55.731139 5729 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6994105f778e464f95640f88841736cc (127.5.57.61:42765): Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:55.731868 5731 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269): Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111)
19:39:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:55.732262 5731 leader_election.cc:302] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cc50be80daea41909bbe4a0f857f0ae7; no voters: 6994105f778e464f95640f88841736cc, 9179aae1af1247599294b6b041d1d97a
19:39:55.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:55.732976 5874 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:39:55.734 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:55.759 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.761 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.763 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.763 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.763 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.763 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.763 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.764 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.766 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:55.919 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:55.920 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:55.922 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.922 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.922 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.922 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.924 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:55.924 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:55.926 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:56.151572 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:56.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:56.151983 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:56.380 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:56.383 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:56.384 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:56.384 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:56.385 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:56.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:56.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:56.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:56.388 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:57.155812 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:57.156181 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:57.219 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:57.221 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:57.223 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:57.224 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:57.224 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:57.224 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:57.224 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:57.224 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:57.226 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:57.399 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:57.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:57.403 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:57.403 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:57.403 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:57.404 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:57.404 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:57.404 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:57.405 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:58.157254 5708 master_runner.cc:186] Error getting master registration for 127.5.57.62:39269: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111),
19:39:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:58.157614 5708 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:39:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:58.159438 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:58.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:58.159693 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:39:58.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:39:58.582 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:39:58.584 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:58.584 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:58.584 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:39:58.585 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:58.585 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:58.585 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:39:58.586 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:39:58.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:58.643543 5882 raft_consensus.cc:482] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:39:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:58.643965 5882 raft_consensus.cc:504] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:39:58.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:58.645464 5882 leader_election.cc:288] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269), 6994105f778e464f95640f88841736cc (127.5.57.61:42765)
19:39:58.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:58.646693 5731 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111) [suppressed 3 similar messages]
19:39:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:58.649911 5731 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269): Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111)
19:39:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:58.650279 5729 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6994105f778e464f95640f88841736cc (127.5.57.61:42765): Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:39:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:58.650563 5729 leader_election.cc:302] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cc50be80daea41909bbe4a0f857f0ae7; no voters: 6994105f778e464f95640f88841736cc, 9179aae1af1247599294b6b041d1d97a
19:39:58.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:39:58.651302 5882 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:39:59.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:59.162683 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:39:59.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:39:59.162992 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:00.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:00.165828 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:00.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:00.166090 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:00.218 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:00.220 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:00.222 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:00.222 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:00.222 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:00.222 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:00.223 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:00.223 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:00.224 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:00.320 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:00.321 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:00.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:00.324 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:00.324 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:00.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:00.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:00.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:00.327 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:01.168668 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:01.168924 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:02.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:02.172189 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:02.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:02.172529 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:40:02 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624402, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:40:02.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:02.810050 5723 init.cc:295] Successfully reacquired a new kerberos TGT
19:40:03.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:03.018334 5890 raft_consensus.cc:482] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:40:03.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:03.018884 5890 raft_consensus.cc:504] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:40:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:03.020457 5890 leader_election.cc:288] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269), 6994105f778e464f95640f88841736cc (127.5.57.61:42765)
19:40:03.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:03.025744 5731 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269): Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111)
19:40:03.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:03.026305 5729 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6994105f778e464f95640f88841736cc (127.5.57.61:42765): Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:40:03.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:03.026557 5729 leader_election.cc:302] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cc50be80daea41909bbe4a0f857f0ae7; no voters: 6994105f778e464f95640f88841736cc, 9179aae1af1247599294b6b041d1d97a
19:40:03.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:03.027040 5890 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:40:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:03.159715 5708 master_runner.cc:186] Error getting master registration for 127.5.57.62:39269: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111),
19:40:03.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:03.160115 5708 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:40:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:03.175276 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:03.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:03.175488 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:03.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:03.518332 4851 debug-util.cc:398] Leaking SignalData structure 0x7b08000bba60 after lost signal to thread 4783
19:40:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:03.519003 4851 debug-util.cc:398] Leaking SignalData structure 0x7b080007cca0 after lost signal to thread 4854
19:40:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:04.178442 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:04.178752 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:04.399 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:04.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:04.402 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:04.402 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:04.402 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:04.404 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:04.404 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:04.405 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:04.405 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:05.101754 4772 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5380 after lost signal to thread 4707
19:40:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:05.102262 4772 debug-util.cc:398] Leaking SignalData structure 0x7b0800081ae0 after lost signal to thread 4775
19:40:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:05.183005 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:05.183333 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:06.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:06.169543 4697 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5e00 after lost signal to thread 4634
19:40:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:06.170034 4697 debug-util.cc:398] Leaking SignalData structure 0x7b08000906c0 after lost signal to thread 4700
19:40:06.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:06.187510 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:06.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:06.187819 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:07.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:07.190743 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:07.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:07.190975 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:08.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:08.065235 5898 raft_consensus.cc:482] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:40:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:08.065604 5898 raft_consensus.cc:504] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:40:08.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:08.067504 5898 leader_election.cc:288] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269), 6994105f778e464f95640f88841736cc (127.5.57.61:42765)
19:40:08.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:08.070178 5731 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111) [suppressed 3 similar messages]
19:40:08.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:08.073230 5731 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269): Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111)
19:40:08.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:08.074710 5729 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6994105f778e464f95640f88841736cc (127.5.57.61:42765): Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:40:08.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:08.075270 5729 leader_election.cc:302] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cc50be80daea41909bbe4a0f857f0ae7; no voters: 6994105f778e464f95640f88841736cc, 9179aae1af1247599294b6b041d1d97a
19:40:08.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:08.075943 5898 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:40:08.079 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:08.081 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:08.081 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:08.081 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:08.082 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:08.083 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:08.083 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:08.083 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:08.086 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:08.119 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:08.121 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:08.122 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:08.122 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:08.123 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:08.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:08.125 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:08.125 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:08.125 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:08.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:08.161994 5708 master_runner.cc:186] Error getting master registration for 127.5.57.62:39269: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111),
19:40:08.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:08.162302 5708 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:40:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:08.195012 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:08.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:08.195384 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:09.198277 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:09.198519 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:09.379 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:09.381 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:09.383 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:09.383 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:09.383 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:09.383 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:09.383 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:09.383 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:09.385 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:10.201198 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:10.201434 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:11.204010 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:11.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:11.204231 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:12.159 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:12.161 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:12.163 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:12.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:12.163 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:12.164 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:12.164 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:12.164 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:12.166 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:12.207063 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:12.207423 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:12.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:40:12 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624412, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:40:12.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:12.812296 5723 init.cc:295] Successfully reacquired a new kerberos TGT
19:40:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:13.165210 5708 master_runner.cc:186] Error getting master registration for 127.5.57.62:39269: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111),
19:40:13.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:13.165555 5708 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:40:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:13.210285 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:13.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:13.210635 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:13.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:13.933658 5910 raft_consensus.cc:482] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:40:13.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:13.934162 5910 raft_consensus.cc:504] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:40:13.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:13.935825 5910 leader_election.cc:288] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269), 6994105f778e464f95640f88841736cc (127.5.57.61:42765)
19:40:13.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:13.937842 5731 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111) [suppressed 1 similar messages]
19:40:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:13.940742 5729 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6994105f778e464f95640f88841736cc (127.5.57.61:42765): Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:40:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:13.941190 5731 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269): Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111)
19:40:13.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:13.941498 5731 leader_election.cc:302] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cc50be80daea41909bbe4a0f857f0ae7; no voters: 6994105f778e464f95640f88841736cc, 9179aae1af1247599294b6b041d1d97a
19:40:13.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:13.942028 5910 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:40:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:14.213555 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:14.213833 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:15.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:15.216616 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:15.216890 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:15.239 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:15.240 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:15.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:15.241 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:15.241 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:15.241 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:15.241 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:15.241 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:15.244 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:16.219914 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:16.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:16.220147 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:17.222815 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:17.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:17.223047 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:18.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:18.167853 5708 master_runner.cc:186] Error getting master registration for 127.5.57.62:39269: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111),
19:40:18.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:18.168265 5708 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:40:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:18.226087 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:18.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:18.226353 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:18.659 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:18.660 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:18.661 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:18.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:18.661 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:18.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:18.662 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:18.662 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:18.664 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:19.229247 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:19.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:19.229554 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:19.779 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:19.779 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:19.780 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:19.780 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:19.780 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:19.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:19.781 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:19.781 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:19.782 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:20.233157 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:20.233446 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:20.819 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:20.820 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:20.821 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:20.821 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:20.821 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:20.821 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:20.821 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:20.822 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:20.824 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:21.236748 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:21.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:21.237084 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:22.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:22.241436 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:22.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:22.241870 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:22.341687 5924 raft_consensus.cc:482] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
19:40:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:22.342200 5924 raft_consensus.cc:504] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9179aae1af1247599294b6b041d1d97a" member_type: VOTER last_known_addr { host: "127.5.57.62" port: 39269 } } peers { permanent_uuid: "6994105f778e464f95640f88841736cc" member_type: VOTER last_known_addr { host: "127.5.57.61" port: 42765 } } peers { permanent_uuid: "cc50be80daea41909bbe4a0f857f0ae7" member_type: VOTER last_known_addr { host: "127.5.57.60" port: 40783 } }
19:40:22.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:22.344347 5924 leader_election.cc:288] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269), 6994105f778e464f95640f88841736cc (127.5.57.61:42765)
19:40:22.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:22.356937 5731 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111) [suppressed 1 similar messages]
19:40:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:22.360054 5729 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 6994105f778e464f95640f88841736cc (127.5.57.61:42765): Network error: Client connection negotiation failed: client connection to 127.5.57.61:42765: connect: Connection refused (error 111)
19:40:22.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:22.360450 5731 leader_election.cc:334] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 9179aae1af1247599294b6b041d1d97a (127.5.57.62:39269): Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111)
19:40:22.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:22.360757 5731 leader_election.cc:302] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: cc50be80daea41909bbe4a0f857f0ae7; no voters: 6994105f778e464f95640f88841736cc, 9179aae1af1247599294b6b041d1d97a
19:40:22.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:22.361397 5924 raft_consensus.cc:2726] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
19:40:22.679 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:22.680 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:22.681 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:22.681 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:22.682 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:22.682 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:22.682 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:22.682 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:22.684 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 28 19:40:22 dist-test-slave-htqb krb5kdc[5361](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790624422, etypes {rep=17 tkt=17 ses=17}, kudu/127.5.57.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
19:40:22.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:22.814507 5723 init.cc:295] Successfully reacquired a new kerberos TGT
19:40:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:23.170606 5708 master_runner.cc:186] Error getting master registration for 127.5.57.62:39269: Network error: Client connection negotiation failed: client connection to 127.5.57.62:39269: connect: Connection refused (error 111),
19:40:23.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:23.171000 5708 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
19:40:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:23.244771 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:23.244993 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:23.759 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:23.760 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:23.761 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:23.761 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:23.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:23.761 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:23.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:23.761 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:23.763 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:24.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:24.247262 5794 catalog_manager.cc:1558] T 00000000000000000000000000000000 P cc50be80daea41909bbe4a0f857f0ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
19:40:24.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260928 19:40:24.247473 5794 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
19:40:24.739 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.61:42765(127.5.57.61:42765)] connecting to peer
19:40:24.740 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.5.57.62:39269(127.5.57.62:39269)] connecting to peer
19:40:24.741 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:24.741 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.61:42765(127.5.57.61:42765)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:24.741 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.61:42765: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765
19:40:24.741 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:24.741 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.5.57.62:39269(127.5.57.62:39269)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:24.742 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.5.57.62:39269: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
19:40:24.743 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269; will retry
19:40:24.748 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29768), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [233ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [236ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [578ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [604ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [606ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [620ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [622ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [625ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [639ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [641ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [643ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [682ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [719ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [720ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [722ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [723ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [743ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [745ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [749ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [782ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [941ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [942ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1395ms] refreshing cache from master, [1396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2234ms] refreshing cache from master, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2242ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2415ms] refreshing cache from master, [2415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2419ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2421ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [3595ms] refreshing cache from master, [3596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [3597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [3598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [3600ms] too many traces: truncated at 100 traces, deferred=Deferred@1592166310(state=PENDING, result=null, callback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863), errback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863)))
19:40:24.753 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.57.62:39269(127.5.57.62:39269)] disconnecting while in state TERMINATED
19:40:24.753 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.57.60:40783(127.5.57.60:40783)] disconnecting while in state READY
19:40:24.754 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.5.57.61:42765(127.5.57.61:42765)] disconnecting while in state TERMINATED
19:40:24.758 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:40:24.760 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.5.57.60:40783(127.5.57.60:40783)] handling channelInactive
19:40:24.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.5.57.60:40783(127.5.57.60:40783)] cleaning up while in state READY due to: connection closed
19:40:24.761 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:40:24.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:24.765180 5348 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 5708
19:40:25.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:40:25Z chronyd exiting
19:40:27.002 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
19:40:27.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:27.005173 4621 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 4905
19:40:27.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:27.031606 4621 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 5060
19:40:27.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:27.058964 4621 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 5202
19:40:27.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:27.094146 4621 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 4633
19:40:27.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:27.127481 4621 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 4703
19:40:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260928 19:40:27.161540 4621 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9OVAZw/build/tsan/bin/kudu with pid 4778
19:40:28.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-28T19:40:28Z chronyd exiting
19:40:29.393 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29768), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [233ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [236ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [578ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [604ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [606ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [620ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [622ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [625ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [639ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [641ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [643ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [682ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [719ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [720ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [722ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [723ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [743ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [745ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [749ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [782ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [941ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [942ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1395ms] refreshing cache from master, [1396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2234ms] refreshing cache from master, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2242ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2415ms] refreshing cache from master, [2415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2419ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2421ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [3595ms] refreshing cache from master, [3596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [3597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [3598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [3600ms] too many traces: truncated at 100 traces, deferred=Deferred@1592166310(state=PENDING, result=null, callback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863), errback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.57.62:39269
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29768), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [233ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [236ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [578ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [604ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [606ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [620ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [622ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [625ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [639ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [641ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [643ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [682ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [719ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [720ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [722ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [723ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [743ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [745ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [749ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [782ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [941ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [942ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1395ms] refreshing cache from master, [1396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2234ms] refreshing cache from master, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2242ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2415ms] refreshing cache from master, [2415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2419ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2421ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [3595ms] refreshing cache from master, [3596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [3597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [3598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [3600ms] too many traces: truncated at 100 traces, deferred=Deferred@1592166310(state=PENDING, result=null, callback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863), errback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.57.62:39269
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
19:40:29.864 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
19:40:29.865 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29768), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [233ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [236ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [578ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [604ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [606ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [620ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [622ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [625ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [639ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [641ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [643ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [682ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [719ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [720ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [722ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [723ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [743ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [745ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [749ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [782ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [941ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [942ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1395ms] refreshing cache from master, [1396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2234ms] refreshing cache from master, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2242ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2415ms] refreshing cache from master, [2415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2419ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2421ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [3595ms] refreshing cache from master, [3596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [3597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [3598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [3600ms] too many traces: truncated at 100 traces, deferred=Deferred@1592166310(state=PENDING, result=null, callback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863), errback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.57.62:39269
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29768), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [174ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [233ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [236ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [578ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [582ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [602ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [604ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [606ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [607ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [620ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [622ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [623ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [625ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [639ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [641ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [643ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [644ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [680ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [682ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [715ms] refreshing cache from master, [715ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [719ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [720ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [722ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [723ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [735ms] refreshing cache from master, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [743ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [745ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [749ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [750ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [775ms] refreshing cache from master, [775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [779ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [781ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [782ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [934ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [938ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [941ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [942ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1395ms] refreshing cache from master, [1396ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2234ms] refreshing cache from master, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2235ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2237ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2240ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2242ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2242ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2415ms] refreshing cache from master, [2415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [2416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [2417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [2419ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.61:42765: Network error: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.62:39269: Network error: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [2420ms] Sub RPC ConnectToMaster: received response from server master-127.5.57.60:40783: OK, [2421ms] delaying RPC due to: Service unavailable: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269, [3595ms] refreshing cache from master, [3596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.61:42765, [3597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.60:40783, [3598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.5.57.62:39269, [3600ms] too many traces: truncated at 100 traces, deferred=Deferred@1592166310(state=PENDING, result=null, callback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863), errback=(continuation of Deferred@372388418 after retry RPC after error@427354872) -> (continuation of Deferred@129441429 after retry RPC after error@1829139058) -> (continuation of Deferred@1780612530 after retry RPC after error@877716772) -> (continuation of Deferred@195823245 after retry RPC after error@1432842344) -> (continuation of Deferred@810474267 after retry RPC after error@50622963) -> (continuation of Deferred@1761056624 after retry RPC after error@108879414) -> (continuation of Deferred@1076747233 after retry RPC after error@1611051518) -> (continuation of Deferred@874444441 after retry RPC after error@1493074626) -> (continuation of Deferred@1981987055 after retry RPC after error@595488404) -> (continuation of Deferred@1620573788 after retry RPC after error@1280542750) -> (continuation of Deferred@794141883 after retry RPC after error@2140108043) -> (continuation of Deferred@540344760 after retry RPC after error@1895202520) -> (continuation of Deferred@1921285121 after retry RPC after error@1990686839) -> (continuation of Deferred@2123861978 after retry RPC after error@1028931182) -> (continuation of Deferred@1818311089 after retry RPC after error@558399626) -> (continuation of Deferred@1667441190 after retry RPC after error@2144913258) -> (continuation of Deferred@445765009 after retry RPC after error@2134399641) -> (continuation of Deferred@87045450 after retry RPC after error@1013596433) -> (continuation of Deferred@675493456 after retry RPC after error@353525075) -> (continuation of Deferred@286040286 after retry RPC after error@2139101479) -> (continuation of Deferred@1321226545 after retry RPC after error@1237284223) -> (continuation of Deferred@1027679942 after retry RPC after error@1115117928) -> (continuation of Deferred@250794446 after retry RPC after error@1012156898) -> (continuation of Deferred@144129976 after retry RPC after error@1884527579) -> (continuation of Deferred@1756842174 after retry RPC after error@1991382803) -> (continuation of Deferred@1198103682 after retry RPC after error@721967863)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.5.57.61:42765,127.5.57.60:40783,127.5.57.62:39269) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.61:42765(127.5.57.61:42765): Connection refused: /127.5.57.61:42765,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.5.57.62:39269(127.5.57.62:39269): Connection refused: /127.5.57.62:39269
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.5.57.62:39269
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)