03:42:16.825 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:42:16.837 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskGaJiXF/test-tmp
03:42:16.840 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:42:16.843 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskGaJiXF/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:42:17.051 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:42:17.237 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:42:17.238 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:42:17.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:17.241391 11538 mini_oidc.cc:152] Starting JWKS server
03:42:17.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:17.245642 11538 webserver.cc:460] Webserver started at http://127.0.0.1:34431/ using document root <none> and password file <none>
03:42:17.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:17.246143 11538 mini_oidc.cc:189] Starting OIDC Discovery server
03:42:17.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:17.247956 11538 webserver.cc:460] Webserver started at http://127.0.0.1:33139/ using document root <none> and password file <none>
03:42:17.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:42:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:42:17.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:42:17Z Disabled control of system clock
03:42:17.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:17.300479 11538 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/wal
03:42:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/data
03:42:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/logs
03:42:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/data/info.pb
03:42:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:17.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.190:37279
03:42:17.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.190
03:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:17.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:17.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34431/jwks.json with env {}
03:42:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.606196 11550 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:17.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.606751 11550 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:17.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.607412 11550 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:17.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.638031 11550 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:17.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.638337 11550 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:17.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.638523 11550 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:17.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.638700 11550 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:17.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.639029 11550 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:17.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.639313 11550 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34431/jwks.json
03:42:17.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:17.674831 11550 master_runner.cc:386] Master server non-default flags:
03:42:17.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:17.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/data
03:42:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/wal
03:42:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:17.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.190:37279
03:42:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:17.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34431/jwks.json
03:42:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/data/info.pb
03:42:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.190
03:42:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:17.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/logs
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:17.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:17.676285 11550 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:17.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:17.678088 11550 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:17.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.695127 11557 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.695278 11559 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:17.695729 11556 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:18.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:18.873381 11558 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:42:18.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.873790 11550 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:42:18.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.877715 11550 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:18.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.880992 11550 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:18.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.882441 11550 hybrid_clock.cc:648] HybridClock initialized: now 1790480538882397 us; error 62 us; skew 500 ppm
03:42:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.883278 11550 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:18.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.899806 11550 webserver.cc:460] Webserver started at http://127.11.68.190:36037/ using document root <none> and password file <none>
03:42:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.900804 11550 fs_manager.cc:359] Metadata directory not provided
03:42:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.900996 11550 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:18.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.901372 11550 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:18.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.906297 11550 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/data/instance:
03:42:18.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0dad1ee6f56c4d18b887d071ce74d240"
03:42:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:18 on dist-test-slave-4tf8"
03:42:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.907497 11550 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/wal/instance:
03:42:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0dad1ee6f56c4d18b887d071ce74d240"
03:42:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:18 on dist-test-slave-4tf8"
03:42:18.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.914734 11550 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
03:42:18.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.920243 11566 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:18.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.921496 11550 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:42:18.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.921963 11550 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/wal
03:42:18.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0dad1ee6f56c4d18b887d071ce74d240"
03:42:18.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:18 on dist-test-slave-4tf8"
03:42:18.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.922448 11550 fs_report.cc:383] FS layout report
03:42:18.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:18.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/wal
03:42:18.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/wal
03:42:18.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/data/data
03:42:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:18.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:18.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:18.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.956091 11550 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.957703 11550 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:18.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:18.958217 11550 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:19.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.026870 11550 rpc_server.cc:307] RPC server started. Bound to: 127.11.68.190:37279
03:42:19.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.026964 11617 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.68.190:37279 every 8 connection(s)
03:42:19.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.029816 11550 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/data/info.pb
03:42:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.034962 11618 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:42:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.037319 11538 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 11550
03:42:19.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.037757 11538 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-0/wal/instance
03:42:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.041899 11538 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/wal
03:42:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/data
03:42:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:19.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/logs
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/data/info.pb
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:19.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:19.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.189:38713
03:42:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.189
03:42:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:19.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:19.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:19.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:19.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:19.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34431/jwks.json with env {}
03:42:19.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.048801 11618 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } has no permanent_uuid. Determining permanent_uuid...
03:42:19.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.079823 11618 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } has no permanent_uuid. Determining permanent_uuid...
03:42:19.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.082062 11570 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.68.189:38713: connect: Connection refused (error 111)
03:42:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.086159 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.189:38713: Network error: Client connection negotiation failed: client connection to 127.11.68.189:38713: connect: Connection refused (error 111)
03:42:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.135638 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } attempt: 1
03:42:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.139194 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.189:38713: Network error: Client connection negotiation failed: client connection to 127.11.68.189:38713: connect: Connection refused (error 111)
03:42:19.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.207587 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } attempt: 2
03:42:19.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.211145 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.189:38713: Network error: Client connection negotiation failed: client connection to 127.11.68.189:38713: connect: Connection refused (error 111)
03:42:19.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.302649 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } attempt: 3
03:42:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.306466 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.189:38713: Network error: Client connection negotiation failed: client connection to 127.11.68.189:38713: connect: Connection refused (error 111)
03:42:19.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.355115 11620 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.355804 11620 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:19.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.356354 11620 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.387004 11620 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.387320 11620 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.387511 11620 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:19.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.387683 11620 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.387979 11620 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:19.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.388135 11620 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34431/jwks.json
03:42:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.422946 11620 master_runner.cc:386] Master server non-default flags:
03:42:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:19.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/data
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/wal
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.189:38713
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:19.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34431/jwks.json
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/data/info.pb
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.189
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:19.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/logs
03:42:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.424296 11620 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.425952 11620 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.442492 11629 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:19.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.449908 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } attempt: 4
03:42:19.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.453359 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.189:38713: Network error: Client connection negotiation failed: client connection to 127.11.68.189:38713: connect: Connection refused (error 111)
03:42:19.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.752771 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } attempt: 5
03:42:19.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.757606 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.189:38713: Network error: Client connection negotiation failed: client connection to 127.11.68.189:38713: connect: Connection refused (error 111)
03:42:20.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.305066 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } attempt: 6
03:42:20.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:20.312800 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.189:38713: Network error: Client connection negotiation failed: client connection to 127.11.68.189:38713: connect: Connection refused (error 111)
03:42:20.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.442649 11630 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:20.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:19.444592 11620 server_base.cc:1034] running on GCE node
03:42:20.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:19.442639 11632 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:20.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.631774 11620 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:20.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.635479 11620 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:20.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.637020 11620 hybrid_clock.cc:648] HybridClock initialized: now 1790480540636969 us; error 61 us; skew 500 ppm
03:42:20.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.638327 11620 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:20.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.651944 11620 webserver.cc:460] Webserver started at http://127.11.68.189:35327/ using document root <none> and password file <none>
03:42:20.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.653062 11620 fs_manager.cc:359] Metadata directory not provided
03:42:20.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.653322 11620 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:20.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.653834 11620 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:20.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.658879 11620 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/data/instance:
03:42:20.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dad884ce7f94682926654178421f9e5"
03:42:20.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:20 on dist-test-slave-4tf8"
03:42:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.660275 11620 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/wal/instance:
03:42:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dad884ce7f94682926654178421f9e5"
03:42:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:20 on dist-test-slave-4tf8"
03:42:20.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.667760 11620 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.005s
03:42:20.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.673234 11641 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:20.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.674268 11620 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
03:42:20.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.674633 11620 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/wal
03:42:20.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dad884ce7f94682926654178421f9e5"
03:42:20.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:20 on dist-test-slave-4tf8"
03:42:20.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.675026 11620 fs_report.cc:383] FS layout report
03:42:20.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/wal
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/wal
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/data/data
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:20.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:20.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:20.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.720292 11620 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:20.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.721854 11620 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:20.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.722384 11620 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:20.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.800414 11620 rpc_server.cc:307] RPC server started. Bound to: 127.11.68.189:38713
03:42:20.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.800581 11692 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.68.189:38713 every 8 connection(s)
03:42:20.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.804070 11620 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/data/info.pb
03:42:20.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.809537 11538 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 11620
03:42:20.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.809969 11538 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-1/wal/instance
03:42:20.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.810390 11693 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.813575 11538 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/wal
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/data
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/logs
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/data/info.pb
03:42:20.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.188:37445
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.188
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:20.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:20.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:20.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:20.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:20.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:20.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:20.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34431/jwks.json with env {}
03:42:20.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.826898 11693 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } has no permanent_uuid. Determining permanent_uuid...
03:42:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.856734 11693 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } has no permanent_uuid. Determining permanent_uuid...
03:42:20.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.873662 11693 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } has no permanent_uuid. Determining permanent_uuid...
03:42:20.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:20.876351 11645 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:20.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:20.880568 11693 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:20.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:20.930032 11693 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 1
03:42:20.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:20.934316 11693 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.002771 11693 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 2
03:42:21.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.006824 11693 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.098242 11693 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 3
03:42:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.102509 11693 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.142513 11695 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.143055 11695 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.143713 11695 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.173741 11695 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.174019 11695 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.174212 11695 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.174526 11695 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.174834 11695 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.175082 11695 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34431/jwks.json
03:42:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.211079 11695 master_runner.cc:386] Master server non-default flags:
03:42:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/data
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/wal
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.188:37445
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34431/jwks.json
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/data/info.pb
03:42:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.188
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/logs
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.212473 11695 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:21.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.214277 11695 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.235365 11706 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:21.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.245941 11693 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 4
03:42:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.250195 11693 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.373252 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } attempt: 7
03:42:21.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.386803 11618 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } has no permanent_uuid. Determining permanent_uuid...
03:42:21.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.392062 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.450706 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 1
03:42:21.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.455829 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.537354 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 2
03:42:21.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.541682 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.549630 11693 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 5
03:42:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.553097 11693 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.627142 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 3
03:42:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.632123 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:21.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.772500 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 4
03:42:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.776173 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:22.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.059778 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 5
03:42:22.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:22.064438 11618 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.100739 11693 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 6
03:42:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:22.108651 11693 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.68.188:37445: Network error: Client connection negotiation failed: client connection to 127.11.68.188:37445: connect: Connection refused (error 111)
03:42:22.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.236871 11705 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:21.238243 11695 server_base.cc:1034] running on GCE node
03:42:22.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:21.240448 11708 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:22.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.446273 11695 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:22.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.449687 11695 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:22.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.451125 11695 hybrid_clock.cc:648] HybridClock initialized: now 1790480542451088 us; error 62 us; skew 500 ppm
03:42:22.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.451979 11695 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.458747 11695 webserver.cc:460] Webserver started at http://127.11.68.188:38927/ using document root <none> and password file <none>
03:42:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.459678 11695 fs_manager.cc:359] Metadata directory not provided
03:42:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.459872 11695 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.460294 11695 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.467366 11695 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/data/instance:
03:42:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "542031ad92dc45c89f6b8427f66d256d"
03:42:22.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:22 on dist-test-slave-4tf8"
03:42:22.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.468909 11695 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/wal/instance:
03:42:22.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "542031ad92dc45c89f6b8427f66d256d"
03:42:22.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:22 on dist-test-slave-4tf8"
03:42:22.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.478003 11695 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
03:42:22.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.483891 11720 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.485046 11695 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:42:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.485338 11695 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/wal
03:42:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "542031ad92dc45c89f6b8427f66d256d"
03:42:22.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:22 on dist-test-slave-4tf8"
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.485653 11695 fs_report.cc:383] FS layout report
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/wal
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/wal
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/data/data
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:22.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:22.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.528260 11695 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:22.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.530104 11695 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:22.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.530666 11695 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.600225 11695 rpc_server.cc:307] RPC server started. Bound to: 127.11.68.188:37445
03:42:22.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.600369 11771 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.68.188:37445 every 8 connection(s)
03:42:22.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.602752 11695 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/data/info.pb
03:42:22.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.608733 11772 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:42:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.611924 11538 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 11695
03:42:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.612390 11538 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/master-2/wal/instance
03:42:22.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.616938 11618 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 6
03:42:22.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.626262 11772 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } has no permanent_uuid. Determining permanent_uuid...
03:42:22.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.671108 11772 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } has no permanent_uuid. Determining permanent_uuid...
03:42:22.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.678098 11618 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: Bootstrap starting.
03:42:22.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.685225 11618 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: Neither blocks nor log segments found. Creating new log.
03:42:22.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.685535 11772 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } has no permanent_uuid. Determining permanent_uuid...
03:42:22.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.689767 11618 log.cc:824] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: Log is configured to *not* fsync() on all Append() calls
03:42:22.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.695132 11618 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: No bootstrap required, opened a new log
03:42:22.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.705365 11772 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d: Bootstrap starting.
03:42:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.713852 11772 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d: Neither blocks nor log segments found. Creating new log.
03:42:22.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.716004 11772 log.cc:824] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d: Log is configured to *not* fsync() on all Append() calls
03:42:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.720456 11772 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d: No bootstrap required, opened a new log
03:42:22.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.721422 11618 raft_consensus.cc:348] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:22.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.722041 11618 raft_consensus.cc:374] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:42:22.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.722332 11618 raft_consensus.cc:729] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dad1ee6f56c4d18b887d071ce74d240, State: Initialized, Role: FOLLOWER
03:42:22.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.723399 11618 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.726866 11784 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.728062 11784 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [sys.catalog]: This master's current role is: FOLLOWER
03:42:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.728771 11618 sys_catalog.cc:564] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [sys.catalog]: configured and running, proceeding with master startup.
03:42:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.744490 11772 raft_consensus.cc:348] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.745128 11772 raft_consensus.cc:374] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:42:22.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.745374 11772 raft_consensus.cc:729] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 542031ad92dc45c89f6b8427f66d256d, State: Initialized, Role: FOLLOWER
03:42:22.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.746503 11772 consensus_queue.cc:260] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.750402 11792 sys_catalog.cc:455] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.751283 11792 sys_catalog.cc:458] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: This master's current role is: FOLLOWER
03:42:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.752314 11772 sys_catalog.cc:564] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: configured and running, proceeding with master startup.
03:42:22.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:22.771698 11798 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:22.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:22.772877 11798 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:22.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:22.785722 11726 tablet.cc:2369] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:42:22.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:22.803144 11808 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:22.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:22.803886 11808 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:22.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.769593 11550 master_runner.cc:186] Error getting master registration for 127.11.68.189:38713: OK, instance_id { permanent_uuid: "2dad884ce7f94682926654178421f9e5" instance_seqno: 1790480540776448 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:42:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.805944 11550 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:42:22.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.794152 11695 master_runner.cc:186] Error getting master registration for 127.11.68.189:38713: OK, instance_id { permanent_uuid: "2dad884ce7f94682926654178421f9e5" instance_seqno: 1790480540776448 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:42:22.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.839397 11695 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:42:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.889967 11792 raft_consensus.cc:482] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:42:22.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.890420 11792 raft_consensus.cc:504] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:22.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.892912 11792 leader_election.cc:288] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0dad1ee6f56c4d18b887d071ce74d240 (127.11.68.190:37279), 2dad884ce7f94682926654178421f9e5 (127.11.68.189:38713)
03:42:22.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.894264 11593 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "542031ad92dc45c89f6b8427f66d256d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dad1ee6f56c4d18b887d071ce74d240" is_pre_election: true
03:42:22.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.895133 11593 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 542031ad92dc45c89f6b8427f66d256d in term 0.
03:42:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.896530 11724 leader_election.cc:302] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0dad1ee6f56c4d18b887d071ce74d240, 542031ad92dc45c89f6b8427f66d256d; no voters:
03:42:22.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.897430 11792 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 0 FOLLOWER]: Leader pre-election won for term 1
03:42:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.897784 11792 raft_consensus.cc:482] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:42:22.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.898102 11792 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 0 FOLLOWER]: Advancing to term 1
03:42:22.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.904631 11792 raft_consensus.cc:504] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:22.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.906435 11792 leader_election.cc:288] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [CANDIDATE]: Term 1 election: Requested vote from peers 0dad1ee6f56c4d18b887d071ce74d240 (127.11.68.190:37279), 2dad884ce7f94682926654178421f9e5 (127.11.68.189:38713)
03:42:22.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.907197 11593 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "542031ad92dc45c89f6b8427f66d256d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dad1ee6f56c4d18b887d071ce74d240"
03:42:22.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.907774 11593 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [term 0 FOLLOWER]: Advancing to term 1
03:42:22.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.914287 11593 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 542031ad92dc45c89f6b8427f66d256d in term 1.
03:42:22.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.923678 11724 leader_election.cc:302] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 0dad1ee6f56c4d18b887d071ce74d240, 542031ad92dc45c89f6b8427f66d256d; no voters:
03:42:22.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.928519 11792 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 1 FOLLOWER]: Leader election won for term 1
03:42:22.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.939730 11792 raft_consensus.cc:686] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [term 1 LEADER]: Becoming Leader. State: Replica: 542031ad92dc45c89f6b8427f66d256d, State: Running, Role: LEADER
03:42:22.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.940752 11792 consensus_queue.cc:237] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:22.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.960153 11810 sys_catalog.cc:455] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 542031ad92dc45c89f6b8427f66d256d. Latest consensus state: current_term: 1 leader_uuid: "542031ad92dc45c89f6b8427f66d256d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:22.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.961071 11810 sys_catalog.cc:458] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: This master's current role is: LEADER
03:42:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:22.967324 11572 tablet.cc:2369] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:42:22.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.968340 11812 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
03:42:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:22.975721 11812 catalog_manager.cc:1484] Initializing Kudu cluster ID...
03:42:23.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.006924 11593 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [term 1 FOLLOWER]: Refusing update from remote peer 542031ad92dc45c89f6b8427f66d256d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:42:23.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.009363 11810 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:42:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.070905 11784 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 542031ad92dc45c89f6b8427f66d256d. Latest consensus state: current_term: 1 leader_uuid: "542031ad92dc45c89f6b8427f66d256d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.071789 11784 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [sys.catalog]: This master's current role is: FOLLOWER
03:42:23.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.078346 11813 mvcc.cc:205] Tried to move back new op lower bound from 7333808304131248128 to 7333808303895834624. Current Snapshot: MvccSnapshot[applied={T|T < 7333808304131248128}]
03:42:23.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.089411 11792 sys_catalog.cc:455] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "542031ad92dc45c89f6b8427f66d256d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:23.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.092110 11792 sys_catalog.cc:458] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: This master's current role is: LEADER
03:42:23.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.092043 11784 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "542031ad92dc45c89f6b8427f66d256d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:23.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.092777 11784 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240 [sys.catalog]: This master's current role is: FOLLOWER
03:42:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.109933 11812 catalog_manager.cc:1347] Generated new cluster ID: 4d212ea0435e4670afffe22387c60f1e
03:42:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.110262 11812 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
03:42:23.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.160065 11812 catalog_manager.cc:1370] Generated new certificate authority record
03:42:23.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.162693 11812 catalog_manager.cc:1504] Loading token signing keys...
03:42:23.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.169265 11693 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } attempt: 7
03:42:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.198709 11812 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d: Generated new TSK 0
03:42:23.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.202456 11812 catalog_manager.cc:1514] Initializing in-progress tserver states...
03:42:23.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.213416 11693 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5: Bootstrap starting.
03:42:23.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.223887 11693 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5: Neither blocks nor log segments found. Creating new log.
03:42:23.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.226585 11693 log.cc:824] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5: Log is configured to *not* fsync() on all Append() calls
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.226562 11538 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/wal
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/data
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/logs
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/data/info.pb
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.129:0
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.68.129
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.129
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:42:23.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.245108 11693 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5: No bootstrap required, opened a new log
03:42:23.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.309032 11693 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:23.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.309900 11693 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:42:23.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.310170 11693 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2dad884ce7f94682926654178421f9e5, State: Initialized, Role: FOLLOWER
03:42:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.323590 11693 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:23.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.328790 11822 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:42:23.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.331851 11668 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "542031ad92dc45c89f6b8427f66d256d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2dad884ce7f94682926654178421f9e5" is_pre_election: true
03:42:23.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.332134 11822 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } }
03:42:23.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.331671 11693 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [sys.catalog]: configured and running, proceeding with master startup.
03:42:23.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.337826 11592 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2dad884ce7f94682926654178421f9e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dad1ee6f56c4d18b887d071ce74d240" is_pre_election: true
03:42:23.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.333597 11667 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "542031ad92dc45c89f6b8427f66d256d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2dad884ce7f94682926654178421f9e5"
03:42:23.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.339080 11747 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2dad884ce7f94682926654178421f9e5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "542031ad92dc45c89f6b8427f66d256d" is_pre_election: true
03:42:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.350841 11645 leader_election.cc:302] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2dad884ce7f94682926654178421f9e5; no voters: 0dad1ee6f56c4d18b887d071ce74d240, 542031ad92dc45c89f6b8427f66d256d
03:42:23.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.336545 11822 leader_election.cc:288] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0dad1ee6f56c4d18b887d071ce74d240 (127.11.68.190:37279), 542031ad92dc45c89f6b8427f66d256d (127.11.68.188:37445)
03:42:23.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.356242 11822 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:42:23.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.358045 11666 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [term 0 FOLLOWER]: Advancing to term 1
03:42:23.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.356249 11824 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:23.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.367316 11824 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [sys.catalog]: This master's current role is: FOLLOWER
03:42:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.368259 11666 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [term 1 FOLLOWER]: Refusing update from remote peer 542031ad92dc45c89f6b8427f66d256d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:42:23.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.374919 11810 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [LEADER]: Connected to new peer: Peer: permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:42:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.427822 11824 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 542031ad92dc45c89f6b8427f66d256d. Latest consensus state: current_term: 1 leader_uuid: "542031ad92dc45c89f6b8427f66d256d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.428615 11824 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [sys.catalog]: This master's current role is: FOLLOWER
03:42:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.428223 11821 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "542031ad92dc45c89f6b8427f66d256d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.428941 11821 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5 [sys.catalog]: This master's current role is: FOLLOWER
03:42:23.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.535810 11838 catalog_manager.cc:1259] Loaded cluster ID: 4d212ea0435e4670afffe22387c60f1e
03:42:23.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.537048 11838 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5: loading cluster ID for follower catalog manager: success
03:42:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.546597 11838 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5: acquiring CA information for follower catalog manager: success
03:42:23.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.552871 11838 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 2dad884ce7f94682926654178421f9e5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:42:23.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.778702 11798 catalog_manager.cc:1259] Loaded cluster ID: 4d212ea0435e4670afffe22387c60f1e
03:42:23.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.779346 11798 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: loading cluster ID for follower catalog manager: success
03:42:23.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.787446 11798 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: acquiring CA information for follower catalog manager: success
03:42:23.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:23.806623 11820 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:23.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:23.807626 11820 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:23.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:23.808598 11820 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.821501 11798 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 0dad1ee6f56c4d18b887d071ce74d240: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:42:23.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:23.864544 11820 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:23.865883 11820 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.68.129
03:42:23.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.882972 11844 sys_catalog.cc:455] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "542031ad92dc45c89f6b8427f66d256d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dad1ee6f56c4d18b887d071ce74d240" member_type: VOTER last_known_addr { host: "127.11.68.190" port: 37279 } } peers { permanent_uuid: "2dad884ce7f94682926654178421f9e5" member_type: VOTER last_known_addr { host: "127.11.68.189" port: 38713 } } peers { permanent_uuid: "542031ad92dc45c89f6b8427f66d256d" member_type: VOTER last_known_addr { host: "127.11.68.188" port: 37445 } } }
03:42:23.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.888484 11844 sys_catalog.cc:458] T 00000000000000000000000000000000 P 542031ad92dc45c89f6b8427f66d256d [sys.catalog]: This master's current role is: LEADER
03:42:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.927170 11820 tablet_server_runner.cc:78] Tablet server non-default flags:
03:42:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/data
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/wal
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.129:0
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/data/info.pb
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.129
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.68.129
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/logs
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.928807 11820 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.930913 11820 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:23.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:23.976063 11849 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:25.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:23.984594 11820 server_base.cc:1034] running on GCE node
03:42:25.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:23.985296 11848 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:25.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:23.984187 11852 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:25.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.455349 11820 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:25.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.465526 11820 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:25.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.467267 11820 hybrid_clock.cc:648] HybridClock initialized: now 1790480545467222 us; error 55 us; skew 500 ppm
03:42:25.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.468593 11820 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.499764 11820 webserver.cc:460] Webserver started at http://127.11.68.129:41285/ using document root <none> and password file <none>
03:42:25.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.505439 11820 fs_manager.cc:359] Metadata directory not provided
03:42:25.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.506109 11820 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:25.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.506898 11820 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:25.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.514997 11820 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/data/instance:
03:42:25.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df82d23b8e61426cb417a8764ff69023"
03:42:25.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:25 on dist-test-slave-4tf8"
03:42:25.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.516912 11820 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/wal/instance:
03:42:25.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df82d23b8e61426cb417a8764ff69023"
03:42:25.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:25 on dist-test-slave-4tf8"
03:42:25.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.532373 11820 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.013s sys 0.000s
03:42:25.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.545679 11858 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:25.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.547300 11820 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.005s sys 0.005s
03:42:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.547909 11820 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/wal
03:42:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "df82d23b8e61426cb417a8764ff69023"
03:42:25.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:25 on dist-test-slave-4tf8"
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.548617 11820 fs_report.cc:383] FS layout report
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/wal
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/wal
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/data/data
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:25.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.627190 11820 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:25.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.629479 11820 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:25.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.630329 11820 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:25.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.639698 11820 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:42:25.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.652297 11820 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:42:25.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.652963 11820 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
03:42:25.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.653506 11820 ts_tablet_manager.cc:610] Registered 0 tablets
03:42:25.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.653921 11820 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:42:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.959939 11820 rpc_server.cc:307] RPC server started. Bound to: 127.11.68.129:39287
03:42:25.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.960052 11970 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.68.129:39287 every 8 connection(s)
03:42:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.962491 11820 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/data/info.pb
03:42:25.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.967108 11538 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 11820
03:42:25.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.967614 11538 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-0/wal/instance
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:25.979873 11538 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/wal
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/data
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:25.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/logs
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/data/info.pb
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.130:0
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.68.130
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.130
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:25.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:42:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.010591 11971 heartbeater.cc:347] Connected to a master server at 127.11.68.188:37445
03:42:26.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.011065 11971 heartbeater.cc:464] Registering TS with master...
03:42:26.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.012328 11971 heartbeater.cc:511] Master 127.11.68.188:37445 requested a full tablet report, sending...
03:42:26.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.015734 11737 ts_manager.cc:194] Registered new tserver with Master: df82d23b8e61426cb417a8764ff69023 (127.11.68.129:39287)
03:42:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.018602 11737 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.68.129:54371
03:42:26.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.025552 11972 heartbeater.cc:347] Connected to a master server at 127.11.68.189:38713
03:42:26.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.025871 11972 heartbeater.cc:464] Registering TS with master...
03:42:26.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.026702 11972 heartbeater.cc:511] Master 127.11.68.189:38713 requested a full tablet report, sending...
03:42:26.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.029737 11658 ts_manager.cc:194] Registered new tserver with Master: df82d23b8e61426cb417a8764ff69023 (127.11.68.129:39287)
03:42:26.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.038357 11973 heartbeater.cc:347] Connected to a master server at 127.11.68.190:37279
03:42:26.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.038673 11973 heartbeater.cc:464] Registering TS with master...
03:42:26.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.039373 11973 heartbeater.cc:511] Master 127.11.68.190:37279 requested a full tablet report, sending...
03:42:26.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.042634 11583 ts_manager.cc:194] Registered new tserver with Master: df82d23b8e61426cb417a8764ff69023 (127.11.68.129:39287)
03:42:26.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:26.330435 11976 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:26.330951 11976 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:26.331457 11976 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:26.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:26.363946 11976 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:26.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:26.364842 11976 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.68.130
03:42:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.399286 11976 tablet_server_runner.cc:78] Tablet server non-default flags:
03:42:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:26.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/data
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/wal
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.130:0
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/data/info.pb
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.130
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:26.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.68.130
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/logs
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.400408 11976 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:26.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:26.401861 11976 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:26.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:26.425073 11988 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:27.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.023761 11971 heartbeater.cc:503] Master 127.11.68.188:37445 was elected leader, sending a full tablet report...
03:42:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:26.426357 11987 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:27.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:26.425499 11990 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:27.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:27.828027 11989 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1398 milliseconds
03:42:27.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.828150 11976 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:42:27.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.829440 11976 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:27.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.831802 11976 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:27.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.833182 11976 hybrid_clock.cc:648] HybridClock initialized: now 1790480547833133 us; error 51 us; skew 500 ppm
03:42:27.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.834079 11976 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:27.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.841437 11976 webserver.cc:460] Webserver started at http://127.11.68.130:42857/ using document root <none> and password file <none>
03:42:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.843006 11976 fs_manager.cc:359] Metadata directory not provided
03:42:27.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.843551 11976 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:27.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.844296 11976 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.850306 11976 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/data/instance:
03:42:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aae71e6559514424953d730ba3382175"
03:42:27.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:27 on dist-test-slave-4tf8"
03:42:27.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.851660 11976 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/wal/instance:
03:42:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aae71e6559514424953d730ba3382175"
03:42:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:27 on dist-test-slave-4tf8"
03:42:27.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.858753 11976 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
03:42:27.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.864439 11999 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:27.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.865583 11976 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:42:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.865933 11976 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/wal
03:42:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aae71e6559514424953d730ba3382175"
03:42:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:27 on dist-test-slave-4tf8"
03:42:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.866292 11976 fs_report.cc:383] FS layout report
03:42:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/wal
03:42:27.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/wal
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/data/data
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:27.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:27.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.906148 11976 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.907719 11976 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:27.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.908250 11976 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:27.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.910725 11976 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:42:27.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.914763 11976 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:42:27.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.914980 11976 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:42:27.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.915336 11976 ts_tablet_manager.cc:610] Registered 0 tablets
03:42:27.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:27.915524 11976 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:42:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.057835 11976 rpc_server.cc:307] RPC server started. Bound to: 127.11.68.130:45807
03:42:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.057893 12111 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.68.130:45807 every 8 connection(s)
03:42:28.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.061273 11976 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/data/info.pb
03:42:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.068339 11538 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 11976
03:42:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.069188 11538 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-1/wal/instance
03:42:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.090204 11538 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:28.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/wal
03:42:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/data
03:42:28.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:28.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/logs
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/data/info.pb
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.131:0
03:42:28.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.68.131
03:42:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.131
03:42:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:28.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:42:28.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.122766 12112 heartbeater.cc:347] Connected to a master server at 127.11.68.188:37445
03:42:28.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.123315 12112 heartbeater.cc:464] Registering TS with master...
03:42:28.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.124717 12112 heartbeater.cc:511] Master 127.11.68.188:37445 requested a full tablet report, sending...
03:42:28.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.127679 11737 ts_manager.cc:194] Registered new tserver with Master: aae71e6559514424953d730ba3382175 (127.11.68.130:45807)
03:42:28.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.130344 12113 heartbeater.cc:347] Connected to a master server at 127.11.68.189:38713
03:42:28.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.130745 12113 heartbeater.cc:464] Registering TS with master...
03:42:28.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.131765 12113 heartbeater.cc:511] Master 127.11.68.189:38713 requested a full tablet report, sending...
03:42:28.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.129998 11737 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.68.130:39059
03:42:28.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.134042 12114 heartbeater.cc:347] Connected to a master server at 127.11.68.190:37279
03:42:28.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.134334 12114 heartbeater.cc:464] Registering TS with master...
03:42:28.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.134375 11658 ts_manager.cc:194] Registered new tserver with Master: aae71e6559514424953d730ba3382175 (127.11.68.130:45807)
03:42:28.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.136291 12114 heartbeater.cc:511] Master 127.11.68.190:37279 requested a full tablet report, sending...
03:42:28.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.138850 11583 ts_manager.cc:194] Registered new tserver with Master: aae71e6559514424953d730ba3382175 (127.11.68.130:45807)
03:42:28.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:28.421725 12121 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:28.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:28.422241 12121 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:28.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:28.422744 12121 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:28.453747 12121 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:28.454535 12121 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.68.131
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.489061 12121 tablet_server_runner.cc:78] Tablet server non-default flags:
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.68.148:37653
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/data
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/wal
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.68.131:0
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/data/info.pb
03:42:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.68.131
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.68.190:37279,127.11.68.189:38713,127.11.68.188:37445
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.68.131
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/logs
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.490268 12121 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.491971 12121 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:28.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:28.510313 12131 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:29.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.137187 12112 heartbeater.cc:503] Master 127.11.68.188:37445 was elected leader, sending a full tablet report...
03:42:29.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:28.510318 12129 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:29.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:28.510787 12128 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:29.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:28.512465 12121 server_base.cc:1034] running on GCE node
03:42:29.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.733114 12121 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.735505 12121 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:29.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.736878 12121 hybrid_clock.cc:648] HybridClock initialized: now 1790480549736846 us; error 48 us; skew 500 ppm
03:42:29.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.737612 12121 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:29.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.744271 12121 webserver.cc:460] Webserver started at http://127.11.68.131:44377/ using document root <none> and password file <none>
03:42:29.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.745311 12121 fs_manager.cc:359] Metadata directory not provided
03:42:29.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.745507 12121 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.745885 12121 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.750765 12121 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/data/instance:
03:42:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afb6acb4b04741aaa4b45aa56edc86a2"
03:42:29.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:29 on dist-test-slave-4tf8"
03:42:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.752003 12121 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/wal/instance:
03:42:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afb6acb4b04741aaa4b45aa56edc86a2"
03:42:29.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:29 on dist-test-slave-4tf8"
03:42:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.759291 12121 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
03:42:29.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.765017 12139 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.766209 12121 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:42:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.766500 12121 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/wal
03:42:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afb6acb4b04741aaa4b45aa56edc86a2"
03:42:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:29 on dist-test-slave-4tf8"
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.766803 12121 fs_report.cc:383] FS layout report
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/wal
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/wal
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/data/data
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:29.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.820161 12121 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:29.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.821890 12121 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:29.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.822459 12121 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:29.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.825374 12121 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
03:42:29.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.830073 12121 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:42:29.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.830389 12121 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:42:29.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.830749 12121 ts_tablet_manager.cc:610] Registered 0 tablets
03:42:29.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.830956 12121 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:42:29.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.971539 12121 rpc_server.cc:307] RPC server started. Bound to: 127.11.68.131:44697
03:42:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.971607 12251 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.68.131:44697 every 8 connection(s)
03:42:29.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.974222 12121 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/data/info.pb
03:42:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.981520 11538 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 12121
03:42:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:29.982028 11538 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3086750667736401094/ts-2/wal/instance
03:42:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.012066 12252 heartbeater.cc:347] Connected to a master server at 127.11.68.188:37445
03:42:30.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.012532 12252 heartbeater.cc:464] Registering TS with master...
03:42:30.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.013751 12252 heartbeater.cc:511] Master 127.11.68.188:37445 requested a full tablet report, sending...
03:42:30.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.015069 12253 heartbeater.cc:347] Connected to a master server at 127.11.68.189:38713
03:42:30.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.015444 12253 heartbeater.cc:464] Registering TS with master...
03:42:30.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.016253 12253 heartbeater.cc:511] Master 127.11.68.189:38713 requested a full tablet report, sending...
03:42:30.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.016546 11737 ts_manager.cc:194] Registered new tserver with Master: afb6acb4b04741aaa4b45aa56edc86a2 (127.11.68.131:44697)
03:42:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.019066 11658 ts_manager.cc:194] Registered new tserver with Master: afb6acb4b04741aaa4b45aa56edc86a2 (127.11.68.131:44697)
03:42:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.019207 11737 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.68.131:36087
03:42:30.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.022473 12254 heartbeater.cc:347] Connected to a master server at 127.11.68.190:37279
03:42:30.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.023053 12254 heartbeater.cc:464] Registering TS with master...
03:42:30.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.023854 12254 heartbeater.cc:511] Master 127.11.68.190:37279 requested a full tablet report, sending...
03:42:30.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.026587 11582 ts_manager.cc:194] Registered new tserver with Master: afb6acb4b04741aaa4b45aa56edc86a2 (127.11.68.131:44697)
03:42:30.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.040678 11538 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
03:42:30.052 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:42:30.054 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:42:30.066 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.68.190"
port: 37279
}
bound_http_address {
host: "127.11.68.190"
port: 36037
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.68.189"
port: 38713
}
bound_http_address {
host: "127.11.68.189"
port: 35327
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.68.188"
port: 37445
}
bound_http_address {
host: "127.11.68.188"
port: 38927
}
}
}
03:42:30.070 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:42:30.073 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.11.68.129"
port: 39287
}
bound_http_address {
host: "127.11.68.129"
port: 41285
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.11.68.130"
port: 45807
}
bound_http_address {
host: "127.11.68.130"
port: 42857
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.11.68.131"
port: 44697
}
bound_http_address {
host: "127.11.68.131"
port: 44377
}
}
}
03:42:30.073 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:42:30.247 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:42:30.360 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskGaJiXF/test-tmp
03:42:30.361 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:42:30.361 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskGaJiXF/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:42:30.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:42:30.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:30.725 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:42:30.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.728266 12263 mini_oidc.cc:152] Starting JWKS server
03:42:30.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.731730 12263 webserver.cc:460] Webserver started at http://127.0.0.1:37473/ using document root <none> and password file <none>
03:42:30.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.732120 12263 mini_oidc.cc:189] Starting OIDC Discovery server
03:42:30.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:30.733762 12263 webserver.cc:460] Webserver started at http://127.0.0.1:43467/ using document root <none> and password file <none>
03:42:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:42:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/principal' for realm 'KRBTEST.COM',
03:42:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:42:30.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:30 dist-test-slave-4tf8 krb5kdc[12276](info): setting up network...
03:42:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:42:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:30 dist-test-slave-4tf8 krb5kdc[12276](info): set up 2 sockets
03:42:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:30 dist-test-slave-4tf8 krb5kdc[12276](info): commencing operation
03:42:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:42:31.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:31.024457 12252 heartbeater.cc:503] Master 127.11.68.188:37445 was elected leader, sending a full tablet report...
03:42:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:31.885286 12263 mini_kdc.cc:121] Time spent starting KDC: real 1.127s user 0.009s sys 0.012s
03:42:31.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:42:31.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:42:31.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:42:31.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:42:31.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:42:31.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:42:31.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:42:31.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:42:31.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:42:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:42:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/test-user.keytab.
03:42:32.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/test-user.keytab.
03:42:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:32 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480552, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:32.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:42:32.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:42:32Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
03:42:32.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:42:32Z Disabled control of system clock
03:42:32.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.249.254@KRBTEST.COM; defaulting to no policy
03:42:32.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:42:32.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.249.254@KRBTEST.COM" created.
03:42:32.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:42:32.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.249.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.254.keytab.
03:42:32.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.249.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.254.keytab.
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:32.105662 12263 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/logs
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/info.pb
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:32.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.254:43933
03:42:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.254
03:42:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.254.keytab
03:42:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.254
03:42:32.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:32.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:42:32.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.420418 12292 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:32.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.421217 12292 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:32.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.422145 12292 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:32.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.454471 12292 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:42:32.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.454764 12292 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:32.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.455188 12292 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:32.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.455510 12292 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:32.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.455826 12292 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:32.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.456139 12292 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:32.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.456364 12292 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37473/jwks.json
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:32.494664 12292 master_runner.cc:386] Master server non-default flags:
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.254.keytab
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.254
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:32.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.254:43933
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/info.pb
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.254
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/logs
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:32.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:32.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:32.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:32.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:32.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:32.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:32.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:32.496228 12292 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:32.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:32.497591 12292 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:32.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.516448 12298 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:32.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.901701 11768 debug-util.cc:398] Leaking SignalData structure 0x7b08000c2520 after lost signal to thread 11700
03:42:32.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.902822 11768 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6420 after lost signal to thread 11771
03:42:33.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.516494 12301 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:33.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:32.518472 12292 server_base.cc:1034] running on GCE node
03:42:33.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:32.516521 12299 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:33.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.782701 12292 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:33.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.786088 12292 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:33.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.787552 12292 hybrid_clock.cc:648] HybridClock initialized: now 1790480553787506 us; error 65 us; skew 500 ppm
03:42:33.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.788383 12292 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:33 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480553, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:33.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.794486 12292 init.cc:377] Logged in from keytab as kudu/127.11.249.254@KRBTEST.COM (short username kudu)
03:42:33.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.800787 12292 webserver.cc:460] Webserver started at http://127.11.249.254:36621/ using document root <none> and password file <none>
03:42:33.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.801851 12292 fs_manager.cc:359] Metadata directory not provided
03:42:33.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.802088 12292 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:33.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.802601 12292 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.807871 12292 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/instance:
03:42:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86decab305634c10acef3c81081e9320"
03:42:33.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:33 on dist-test-slave-4tf8"
03:42:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.809016 12292 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal/instance:
03:42:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86decab305634c10acef3c81081e9320"
03:42:33.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:33 on dist-test-slave-4tf8"
03:42:33.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.816648 12292 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
03:42:33.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.822759 12310 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:33.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.824364 12292 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.824730 12292 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86decab305634c10acef3c81081e9320"
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:33 on dist-test-slave-4tf8"
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.825088 12292 fs_report.cc:383] FS layout report
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/data
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:33.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:33.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:33.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.894487 12292 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:33.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.900278 12292 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:33.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:33.900806 12292 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:34.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.000622 12292 rpc_server.cc:307] RPC server started. Bound to: 127.11.249.254:43933
03:42:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.000712 12361 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.249.254:43933 every 8 connection(s)
03:42:34.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.003619 12292 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/info.pb
03:42:34.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.010426 12362 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:42:34.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.011062 12263 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 12292
03:42:34.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.011603 12263 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal/instance
03:42:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.249.253@KRBTEST.COM; defaulting to no policy
03:42:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:42:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.249.253@KRBTEST.COM" created.
03:42:34.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.028373 12362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } has no permanent_uuid. Determining permanent_uuid...
03:42:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:42:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.249.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.253.keytab.
03:42:34.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.249.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.253.keytab.
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.078429 12263 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/logs
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/info.pb
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.253:43427
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.253
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.253.keytab
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.253
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:34.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:42:34.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:34 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480553, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.254@KRBTEST.COM for kudu/127.11.249.254@KRBTEST.COM
03:42:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.118891 12362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } has no permanent_uuid. Determining permanent_uuid...
03:42:34.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.121598 12313 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:34.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.125799 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.253:43427: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:34.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.175338 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } attempt: 1
03:42:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.179733 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.253:43427: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:34.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.248168 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } attempt: 2
03:42:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.253541 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.253:43427: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:34.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.344965 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } attempt: 3
03:42:34.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.349308 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.253:43427: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.449668 12368 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:34.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.450343 12368 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:34.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.451042 12368 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.483384 12368 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:42:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.483669 12368 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.484050 12368 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.484373 12368 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:34.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.484710 12368 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.485034 12368 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.485267 12368 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37473/jwks.json
03:42:34.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.492795 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } attempt: 4
03:42:34.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.497500 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.253:43427: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.524852 12368 master_runner.cc:386] Master server non-default flags:
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.253.keytab
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.253
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.253:43427
03:42:34.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:42:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:42:34.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/info.pb
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.253
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/logs
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:34.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.526335 12368 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:34.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.527678 12368 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:34.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.550669 12375 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:34.796972 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } attempt: 5
03:42:34.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.803861 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.253:43427: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.351433 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } attempt: 6
03:42:35.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:35.362988 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.253:43427: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.552783 12378 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:34.551368 12376 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:35.904995 12377 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1351 milliseconds
03:42:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.905114 12368 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:42:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.906389 12368 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:35.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.909447 12368 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.910880 12368 hybrid_clock.cc:648] HybridClock initialized: now 1790480555910841 us; error 58 us; skew 500 ppm
03:42:35.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.911756 12368 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:35.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:35 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480555, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:35.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.924301 12368 init.cc:377] Logged in from keytab as kudu/127.11.249.253@KRBTEST.COM (short username kudu)
03:42:35.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.929906 12368 webserver.cc:460] Webserver started at http://127.11.249.253:45743/ using document root <none> and password file <none>
03:42:35.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.930840 12368 fs_manager.cc:359] Metadata directory not provided
03:42:35.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.931041 12368 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:35.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.931615 12368 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.936955 12368 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/instance:
03:42:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74d061b544644abab8983f3e0410fc16"
03:42:35.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:35 on dist-test-slave-4tf8"
03:42:35.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.938086 12368 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal/instance:
03:42:35.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74d061b544644abab8983f3e0410fc16"
03:42:35.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:35 on dist-test-slave-4tf8"
03:42:35.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.946373 12368 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
03:42:35.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.952062 12388 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.953187 12368 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:42:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.953462 12368 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74d061b544644abab8983f3e0410fc16"
03:42:35.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:35 on dist-test-slave-4tf8"
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:35.953788 12368 fs_report.cc:383] FS layout report
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/data
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:35.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:36.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.010308 12368 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.016203 12368 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.016604 12368 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:36.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.110368 12368 rpc_server.cc:307] RPC server started. Bound to: 127.11.249.253:43427
03:42:36.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.110477 12439 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.249.253:43427 every 8 connection(s)
03:42:36.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.113034 12368 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/info.pb
03:42:36.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.115394 12263 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 12368
03:42:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.115903 12263 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal/instance
03:42:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.120062 12440 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:42:36.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.11.249.252@KRBTEST.COM; defaulting to no policy
03:42:36.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:42:36.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.11.249.252@KRBTEST.COM" created.
03:42:36.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.137470 12440 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } has no permanent_uuid. Determining permanent_uuid...
03:42:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:42:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.249.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.252.keytab.
03:42:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.11.249.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.252.keytab.
03:42:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.165721 12263 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/logs
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/info.pb
03:42:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.252:33077
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.252
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.252.keytab
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.252
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:42:36.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:36 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480555, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.253@KRBTEST.COM for kudu/127.11.249.254@KRBTEST.COM
03:42:36.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.185946 12440 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } has no permanent_uuid. Determining permanent_uuid...
03:42:36.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:36 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480555, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.253@KRBTEST.COM for kudu/127.11.249.253@KRBTEST.COM
03:42:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.214725 12440 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } has no permanent_uuid. Determining permanent_uuid...
03:42:36.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.216960 12390 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.221539 12440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.271077 12440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 1
03:42:36.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.275673 12440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.344123 12440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 2
03:42:36.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.348254 12440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.423671 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } attempt: 7
03:42:36.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:36 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480553, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.254@KRBTEST.COM for kudu/127.11.249.253@KRBTEST.COM
03:42:36.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.440068 12440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 3
03:42:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.443998 12440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.448635 12362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } has no permanent_uuid. Determining permanent_uuid...
03:42:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.453789 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.498097 12446 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:36.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.498639 12446 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:36.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.499280 12446 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.512236 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 1
03:42:36.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.515753 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.530949 12446 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:42:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.531232 12446 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.531517 12446 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:36.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.531723 12446 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.531919 12446 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.532104 12446 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:36.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.532251 12446 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37473/jwks.json
03:42:36.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.569538 12446 master_runner.cc:386] Master server non-default flags:
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.252.keytab
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.252
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.252:33077
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/info.pb
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.252
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/logs
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.570685 12446 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.571702 12446 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:36.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.587399 12440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 4
03:42:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.591843 12440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.591857 12460 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:36.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.597098 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 2
03:42:36.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.600514 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.685874 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 3
03:42:36.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.690518 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.830878 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 4
03:42:36.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.834761 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:36.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.891254 12440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 5
03:42:36.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.895821 12440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.118247 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 5
03:42:37.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:37.124759 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:37.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.443254 12440 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 6
03:42:37.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:37.450903 12440 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:37.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.677357 12362 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } attempt: 6
03:42:37.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:37.683899 12362 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.11.249.252:33077: Network error: Client connection negotiation failed: client connection to 127.11.249.252:33077: connect: Connection refused (error 111)
03:42:37.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.593262 12458 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:37.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:36.593525 12457 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:37.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:36.593539 12446 server_base.cc:1034] running on GCE node
03:42:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.869313 12446 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.872176 12446 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.873562 12446 hybrid_clock.cc:648] HybridClock initialized: now 1790480557873532 us; error 39 us; skew 500 ppm
03:42:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.874651 12446 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:37.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:37 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480557, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:37.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.881033 12446 init.cc:377] Logged in from keytab as kudu/127.11.249.252@KRBTEST.COM (short username kudu)
03:42:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.887084 12446 webserver.cc:460] Webserver started at http://127.11.249.252:33223/ using document root <none> and password file <none>
03:42:37.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.888281 12446 fs_manager.cc:359] Metadata directory not provided
03:42:37.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.888530 12446 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:37.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.889046 12446 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
03:42:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.893724 12446 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/instance:
03:42:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "995018245ea348158f462c5199407a3c"
03:42:37.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:37 on dist-test-slave-4tf8"
03:42:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.894897 12446 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal/instance:
03:42:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "995018245ea348158f462c5199407a3c"
03:42:37.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:37 on dist-test-slave-4tf8"
03:42:37.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.903023 12446 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
03:42:37.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.909397 12475 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:37.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.910640 12446 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:42:37.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.911058 12446 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:37.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "995018245ea348158f462c5199407a3c"
03:42:37.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:37 on dist-test-slave-4tf8"
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.911514 12446 fs_report.cc:383] FS layout report
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/data
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:37.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:37.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:37.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:37.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:37.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:37.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:37.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:37.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.963678 12446 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.970036 12446 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:37.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:37.970609 12446 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.074200 12446 rpc_server.cc:307] RPC server started. Bound to: 127.11.249.252:33077
03:42:38.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.074282 12526 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.249.252:33077 every 8 connection(s)
03:42:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.077620 12446 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/info.pb
03:42:38.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.082850 12263 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 12446
03:42:38.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.083343 12263 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal/instance
03:42:38.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.088747 12527 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:42:38.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.108268 12527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } has no permanent_uuid. Determining permanent_uuid...
03:42:38.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:38 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480552, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.249.252@KRBTEST.COM
03:42:38.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:38 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480557, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.252@KRBTEST.COM for kudu/127.11.249.254@KRBTEST.COM
03:42:38.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.150604 12527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } has no permanent_uuid. Determining permanent_uuid...
03:42:38.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:38 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480552, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.249.253@KRBTEST.COM
03:42:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:38 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480557, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.252@KRBTEST.COM for kudu/127.11.249.253@KRBTEST.COM
03:42:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.180123 12527 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } has no permanent_uuid. Determining permanent_uuid...
03:42:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:38 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480552, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.249.254@KRBTEST.COM
03:42:38.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:38 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480557, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.252@KRBTEST.COM for kudu/127.11.249.252@KRBTEST.COM
03:42:38.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.202827 12263 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
03:42:38.209 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:38.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.209957 12527 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: Bootstrap starting.
03:42:38.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:42:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.217027 12527 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: Neither blocks nor log segments found. Creating new log.
03:42:38.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.219539 12527 log.cc:824] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: Log is configured to *not* fsync() on all Append() calls
03:42:38.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:42:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.225057 12527 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: No bootstrap required, opened a new log
03:42:38.226 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:42:38.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.249.254"
port: 43933
}
bound_http_address {
host: "127.11.249.254"
port: 36621
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.249.253"
port: 43427
}
bound_http_address {
host: "127.11.249.253"
port: 45743
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.249.252"
port: 33077
}
bound_http_address {
host: "127.11.249.252"
port: 33223
}
}
}
03:42:38.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:42:38.230 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:38.230 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:42:38.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:42:38.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.254128 12527 raft_consensus.cc:348] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.254769 12527 raft_consensus.cc:374] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:42:38.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.255043 12527 raft_consensus.cc:729] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 995018245ea348158f462c5199407a3c, State: Initialized, Role: FOLLOWER
03:42:38.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.255977 12527 consensus_queue.cc:260] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.259202 12538 sys_catalog.cc:455] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } } }
03:42:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.260195 12538 sys_catalog.cc:458] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [sys.catalog]: This master's current role is: FOLLOWER
03:42:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.260963 12527 sys_catalog.cc:564] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [sys.catalog]: configured and running, proceeding with master startup.
03:42:38.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:38 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480558, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:38.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:42:38.271 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:38.274 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc
03:42:38.291 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:42:38.293 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc
03:42:38.298 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:42:38.311 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:42:38.313 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:42:38.314 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:42:38.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.340553 12549 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:42:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:38 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480558, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:38.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:42:38.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.346451 12446 master_runner.cc:186] Error getting master registration for 127.11.249.254:43933: OK, instance_id { permanent_uuid: "86decab305634c10acef3c81081e9320" instance_seqno: 1790480553965053 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:42:38.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.376839 12446 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:42:38.377 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:38.377 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc
03:42:38.379 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:42:38.380 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.249.254:43933
03:42:38.382 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:42:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.382848 12263 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 12292
03:42:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.398178 12538 raft_consensus.cc:482] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:42:38.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.398876 12538 raft_consensus.cc:504] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:38.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.402154 12538 leader_election.cc:288] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86decab305634c10acef3c81081e9320 (127.11.249.254:43933), 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427)
03:42:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.408664 12479 connection.cc:535] client connection to 127.11.249.254:43933 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:42:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.409149 12479 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:42:38.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.413874 12479 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86decab305634c10acef3c81081e9320 (127.11.249.254:43933): Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111)
03:42:38.418 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:38.419 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.249.253:43427
03:42:38.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:42:38.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.420536 12263 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 12368
03:42:38.441 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:38.441 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.11.249.252:33077
03:42:38.441 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:42:38.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.441922 12263 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 12446
03:42:38.465 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:38.465 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.249.254:43933
03:42:38.468 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.470044 12263 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/logs
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/info.pb
03:42:38.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.254:43933
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.254
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36621
03:42:38.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.254.keytab
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.254
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:38.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:42:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.807825 12554 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:38.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.808604 12554 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:38.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.809600 12554 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.849026 12554 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:42:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.849263 12554 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.849526 12554 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.849718 12554 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.849905 12554 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.850085 12554 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:38.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.850220 12554 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37473/jwks.json
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.887600 12554 master_runner.cc:386] Master server non-default flags:
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.254.keytab
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.254
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.254:43933
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/info.pb
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.254
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36621
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/logs
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:38.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:38.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:38.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:38.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:38.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:38.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.888764 12554 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:38.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.889681 12554 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:38.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.906482 12563 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:39.370189 11967 debug-util.cc:398] Leaking SignalData structure 0x7b0800044740 after lost signal to thread 11840
03:42:39.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:39.371114 11967 debug-util.cc:398] Leaking SignalData structure 0x7b08000afc80 after lost signal to thread 11970
03:42:39.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:39.518700 12248 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd8c0 after lost signal to thread 12123
03:42:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:39.520509 12248 debug-util.cc:398] Leaking SignalData structure 0x7b08000afcc0 after lost signal to thread 12251
03:42:40.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.906508 12561 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:40.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:38.908123 12554 server_base.cc:1034] running on GCE node
03:42:40.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:38.906500 12560 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:40.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.382650 12554 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:40.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.386330 12554 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:40.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.388044 12554 hybrid_clock.cc:648] HybridClock initialized: now 1790480560387961 us; error 78 us; skew 500 ppm
03:42:40.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.389346 12554 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:40.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:40 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480560, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:40.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.397511 12554 init.cc:377] Logged in from keytab as kudu/127.11.249.254@KRBTEST.COM (short username kudu)
03:42:40.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.405031 12554 webserver.cc:460] Webserver started at http://127.11.249.254:36621/ using document root <none> and password file <none>
03:42:40.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.406687 12554 fs_manager.cc:359] Metadata directory not provided
03:42:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.406997 12554 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:40.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.419260 12554 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.002s sys 0.005s
03:42:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.426337 12572 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:40.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.427826 12554 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.006s sys 0.000s
03:42:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.428251 12554 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86decab305634c10acef3c81081e9320"
03:42:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:33 on dist-test-slave-4tf8"
03:42:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.432019 12554 fs_report.cc:383] FS layout report
03:42:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:40.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/data
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:40.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:40.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.488030 12554 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:40.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.496335 12554 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:40.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.496933 12554 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:40.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.604207 12623 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.249.254:43933 every 8 connection(s)
03:42:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.604223 12554 rpc_server.cc:307] RPC server started. Bound to: 127.11.249.254:43933
03:42:40.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.610185 12554 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/data/info.pb
03:42:40.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.619244 12263 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 12554
03:42:40.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:40.620 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.249.253:43427
03:42:40.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.622687 12263 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/logs
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/info.pb
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:40.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.253:43427
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.253
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45743
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.253.keytab
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.253
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:40.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:42:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.630657 12624 sys_catalog.cc:263] Verifying existing consensus state
03:42:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 03:42:40.631745 12624 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:42:40.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.632248 12554 master.cc:537] Master@127.11.249.254:43933 shutting down...
03:42:40.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:40.754292 12554 master.cc:552] Master@127.11.249.254:43933 shutdown complete.
03:42:40.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:42:41.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.162827 12626 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.163721 12626 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:41.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.164917 12626 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:41.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.231377 12626 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:42:41.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.231732 12626 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:41.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.232262 12626 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:41.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.232576 12626 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:41.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.232909 12626 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:41.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.233636 12626 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:41.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.234160 12626 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37473/jwks.json
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:41.309566 12626 master_runner.cc:386] Master server non-default flags:
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.253.keytab
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.253
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.253:43427
03:42:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/info.pb
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.253
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45743
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/logs
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:41.311301 12626 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:41.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:41.312788 12626 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:41.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.336855 12634 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:42.730612 12633 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 12626
03:42:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.011883 12626 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.678s user 0.578s sys 1.067s
03:42:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:41.339111 12635 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.012684 12626 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.679s user 0.579s sys 1.067s
03:42:43.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.014599 12637 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:43.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.017325 12636 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1678 milliseconds
03:42:43.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.017550 12626 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:42:43.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.019011 12626 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.021847 12626 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:43.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.023276 12626 hybrid_clock.cc:648] HybridClock initialized: now 1790480563023242 us; error 37 us; skew 500 ppm
03:42:43.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.024181 12626 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:43.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:43 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480563, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:43.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.036517 12626 init.cc:377] Logged in from keytab as kudu/127.11.249.253@KRBTEST.COM (short username kudu)
03:42:43.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.042593 12626 webserver.cc:460] Webserver started at http://127.11.249.253:45743/ using document root <none> and password file <none>
03:42:43.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.044006 12626 fs_manager.cc:359] Metadata directory not provided
03:42:43.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.044284 12626 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.055271 12626 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.005s sys 0.002s
03:42:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.061823 12645 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.063046 12626 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.003s sys 0.003s
03:42:43.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.063489 12626 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "74d061b544644abab8983f3e0410fc16"
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:35 on dist-test-slave-4tf8"
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.065903 12626 fs_report.cc:383] FS layout report
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/data
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:43.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:43.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:43.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.116209 12626 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:43.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.122098 12626 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:43.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.122611 12626 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.193603 12626 rpc_server.cc:307] RPC server started. Bound to: 127.11.249.253:43427
03:42:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.193740 12697 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.249.253:43427 every 8 connection(s)
03:42:43.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.196489 12626 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/data/info.pb
03:42:43.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.197837 12263 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 12626
03:42:43.199 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:43.199 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.11.249.252:33077
03:42:43.200 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.201839 12263 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:43.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/logs
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/info.pb
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.252:33077
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.252
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33223
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.252.keytab
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.252
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:42:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.210703 12698 sys_catalog.cc:263] Verifying existing consensus state
03:42:43.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260927 03:42:43.211484 12698 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:42:43.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.211856 12626 master.cc:537] Master@127.11.249.253:43427 shutting down...
03:42:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.274808 12626 master.cc:552] Master@127.11.249.253:43427 shutdown complete.
03:42:43.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
03:42:43.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.526315 12700 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:42:43.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.526839 12700 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:42:43.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.527508 12700 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:42:43.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.560356 12700 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:42:43.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.560716 12700 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:42:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.561015 12700 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:42:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.561214 12700 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:42:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.561403 12700 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:42:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.561589 12700 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:42:43.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.561720 12700 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37473/jwks.json
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.599072 12700 master_runner.cc:386] Master server non-default flags:
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.249.212:44473
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:42:43.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/krb5kdc/kudu_127.11.249.252.keytab
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.11.249.252
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.249.252:33077
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37473/jwks.json
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/info.pb
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.249.252
03:42:43.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33223
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/logs
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 27 Sep 2026 02:49:50 UTC on ccycloud.quasar-twsplx.root.comops.site
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675617
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.600276 12700 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:42:43.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.601397 12700 file_cache.cc:493] Constructed file cache file cache with capacity 419430
03:42:43.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.620083 12707 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:44.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.620141 12706 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:44.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:43.622380 12700 server_base.cc:1034] running on GCE node
03:42:44.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:43.621007 12709 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:42:44.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.864225 12700 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:42:44.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.866930 12700 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:42:44.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.868471 12700 hybrid_clock.cc:648] HybridClock initialized: now 1790480564868457 us; error 62 us; skew 500 ppm
03:42:44.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.869259 12700 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:42:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:44 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480564, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:44.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.875592 12700 init.cc:377] Logged in from keytab as kudu/127.11.249.252@KRBTEST.COM (short username kudu)
03:42:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.880149 12700 webserver.cc:460] Webserver started at http://127.11.249.252:33223/ using document root <none> and password file <none>
03:42:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.881029 12700 fs_manager.cc:359] Metadata directory not provided
03:42:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.881219 12700 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:42:44.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.889431 12700 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
03:42:44.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.893908 12718 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:42:44.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.895282 12700 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.895645 12700 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data,/tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "995018245ea348158f462c5199407a3c"
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-27 03:42:37 on dist-test-slave-4tf8"
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.897924 12700 fs_report.cc:383] FS layout report
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/wal
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/data
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:42:44.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:42:44.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.939980 12700 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
03:42:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.946172 12700 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:42:44.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:44.946652 12700 kserver.cc:163] Server-wide thread pool size limit: 3276
03:42:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.019726 12700 rpc_server.cc:307] RPC server started. Bound to: 127.11.249.252:33077
03:42:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.019855 12769 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.11.249.252:33077 every 8 connection(s)
03:42:45.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.023020 12700 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskGaJiXF/test-tmp/mini-kudu-cluster3043311034129134573/master-2/data/info.pb
03:42:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.024644 12263 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu as pid 12700
03:42:45.025 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:42:45.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.038543 12770 sys_catalog.cc:263] Verifying existing consensus state
03:42:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.047941 12770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: Bootstrap starting.
03:42:45.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.093375 12770 log.cc:824] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: Log is configured to *not* fsync() on all Append() calls
03:42:45.094 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:45.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.098335 12770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:42:45.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.099491 12770 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: Bootstrap complete.
03:42:45.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.135231 12770 raft_consensus.cc:348] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:45.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.138725 12770 raft_consensus.cc:374] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:42:45.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.139128 12770 raft_consensus.cc:729] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 995018245ea348158f462c5199407a3c, State: Initialized, Role: FOLLOWER
03:42:45.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.140192 12770 consensus_queue.cc:260] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:45.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.143366 12774 sys_catalog.cc:455] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } } }
03:42:45.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.144495 12774 sys_catalog.cc:458] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [sys.catalog]: This master's current role is: FOLLOWER
03:42:45.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.145290 12770 sys_catalog.cc:564] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [sys.catalog]: configured and running, proceeding with master startup.
03:42:45.164 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:45.165 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.11.249.252:33077(127.11.249.252:33077)] connecting to peer
03:42:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:45.181874 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:45.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:45.182310 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:45.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:45.201797 12724 tablet.cc:2369] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:42:45.217 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.218 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.193859 12700 master_runner.cc:186] Error getting master registration for 127.11.249.254:43933: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111),
03:42:45.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.220634 12700 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:42:45.226 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.231 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.232 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.233 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.233 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.11.249.252:33077(127.11.249.252:33077)] Successfully connected to peer
03:42:45.310 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:42:45.320 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.11.249.252
03:42:45.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.409205 12774 raft_consensus.cc:482] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:42:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.409756 12774 raft_consensus.cc:504] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:45.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.412900 12774 leader_election.cc:288] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86decab305634c10acef3c81081e9320 (127.11.249.254:43933), 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427)
03:42:45.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:45.416108 12722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111)
03:42:45.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:45.420671 12722 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86decab305634c10acef3c81081e9320 (127.11.249.254:43933): Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111)
03:42:45.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:45.422151 12721 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427): Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.422587 12721 leader_election.cc:302] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 995018245ea348158f462c5199407a3c; no voters: 74d061b544644abab8983f3e0410fc16, 86decab305634c10acef3c81081e9320
03:42:45.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:45.423421 12774 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:42:45.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:45 dist-test-slave-4tf8 krb5kdc[12276](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480558, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.11.249.252@KRBTEST.COM
03:42:45.580 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xbceab853, L:/127.0.0.1:37700 - R:/127.11.249.252:33077] using SASL/GSSAPI
03:42:45.596 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:45.613 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:45.615 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:45.616 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.616 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.621 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.622 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.622 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.625 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:45.632 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:45.633 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:45.634 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.634 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.635 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.641 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.641 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.641 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.642 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:45.651 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:45.653 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:45.654 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.654 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.655 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.657 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.657 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.660 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:45.691 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:45.693 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:45.695 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.695 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.696 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.697 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.697 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.698 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.700 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:45.732 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:45.733 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:45.735 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.735 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.735 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.736 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.736 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.736 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.741 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:45.792 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:45.794 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:45.795 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.795 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.800 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.806 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.806 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.807 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.807 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:45.832 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:45.834 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:45.835 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.835 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.837 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:45.839 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.839 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.840 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:45.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:46.092 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:46.093 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:46.094 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.094 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.095 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.103 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.103 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.103 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.104 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:46.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:46.185537 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:46.185832 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:46.392 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:46.394 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:46.395 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.395 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.397 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.397 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.397 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.398 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.402 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:46.912 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:46.913 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:46.914 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.914 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.914 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:46.917 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.917 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.917 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:46.919 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:46.999121 12795 raft_consensus.cc:482] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:42:47.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:46.999608 12795 raft_consensus.cc:504] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:47.001286 12795 leader_election.cc:288] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86decab305634c10acef3c81081e9320 (127.11.249.254:43933), 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427)
03:42:47.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:47.007870 12721 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427): Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:47.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:47.008805 12722 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86decab305634c10acef3c81081e9320 (127.11.249.254:43933): Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111)
03:42:47.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:47.009195 12722 leader_election.cc:302] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 995018245ea348158f462c5199407a3c; no voters: 74d061b544644abab8983f3e0410fc16, 86decab305634c10acef3c81081e9320
03:42:47.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:47.009845 12795 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:42:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:47.188840 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:47.189148 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:47.211 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:47.213 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:47.213 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:47.214 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:47.214 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:47.216 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:47.216 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:47.216 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:47.218 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:48.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:48.192104 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:48.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:48.192409 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:49.051323 12802 raft_consensus.cc:482] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:42:49.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:49.051735 12802 raft_consensus.cc:504] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:49.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:49.053400 12802 leader_election.cc:288] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86decab305634c10acef3c81081e9320 (127.11.249.254:43933), 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427)
03:42:49.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:49.059281 12722 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86decab305634c10acef3c81081e9320 (127.11.249.254:43933): Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111)
03:42:49.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:49.059700 12721 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427): Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:49.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:49.060093 12721 leader_election.cc:302] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 995018245ea348158f462c5199407a3c; no voters: 74d061b544644abab8983f3e0410fc16, 86decab305634c10acef3c81081e9320
03:42:49.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:49.060746 12802 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:42:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:49.195369 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:49.195622 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:50.052 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:50.054 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:50.055 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:50.055 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:50.055 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:50.057 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:50.057 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:50.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:50.059 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:50.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:50.198374 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:50.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:50.198611 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:50.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:50.223412 12700 master_runner.cc:186] Error getting master registration for 127.11.249.254:43933: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111),
03:42:50.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:50.223748 12700 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:42:51.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:51.201400 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:51.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:51.201684 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:51.812 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:51.813 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:51.815 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:51.815 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:51.816 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:51.816 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:51.816 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:51.816 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:51.819 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:52.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:52.204694 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:52.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:52.205024 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:52.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:52.646907 11967 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5ac0 after lost signal to thread 11840
03:42:52.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:52.647967 11967 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8fa0 after lost signal to thread 11970
03:42:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:53.208082 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:53.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:53.208384 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:53.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:53.393064 12810 raft_consensus.cc:482] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:42:53.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:53.393499 12810 raft_consensus.cc:504] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:53.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:53.395143 12810 leader_election.cc:288] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86decab305634c10acef3c81081e9320 (127.11.249.254:43933), 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427)
03:42:53.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:53.397179 12721 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111) [suppressed 5 similar messages]
03:42:53.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:53.399690 12722 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86decab305634c10acef3c81081e9320 (127.11.249.254:43933): Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111)
03:42:53.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:53.400148 12721 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427): Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:53.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:53.400527 12721 leader_election.cc:302] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 995018245ea348158f462c5199407a3c; no voters: 74d061b544644abab8983f3e0410fc16, 86decab305634c10acef3c81081e9320
03:42:53.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:53.401134 12810 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:42:53.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:53.773 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:53.775 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:53.775 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:53.775 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:53.776 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:53.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:53.777 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:53.779 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:54.211621 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:54.212024 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:54.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:42:54 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480574, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:42:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:54.879446 12714 init.cc:295] Successfully reacquired a new kerberos TGT
03:42:55.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:55.214919 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:55.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:55.215215 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:55.226737 12700 master_runner.cc:186] Error getting master registration for 127.11.249.254:43933: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111),
03:42:55.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:55.227140 12700 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:42:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:56.217916 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:56.218210 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:56.472 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:56.473 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:56.474 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:56.474 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:56.475 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:56.475 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:56.475 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:56.475 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:56.478 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:57.221206 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:57.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:57.221509 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:58.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:58.224368 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:58.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:58.224650 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:58.272 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:58.273 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:58.275 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:58.275 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:58.275 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:58.276 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:58.276 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:58.276 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:58.279 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:58.672 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:42:58.673 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:42:58.674 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:58.674 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:58.674 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:42:58.676 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:58.676 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:58.676 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:42:58.678 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:42:59.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:59.227593 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:42:59.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:59.227877 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:42:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:59.558480 12824 raft_consensus.cc:482] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:42:59.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:59.558993 12824 raft_consensus.cc:504] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:42:59.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:59.560824 12824 leader_election.cc:288] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86decab305634c10acef3c81081e9320 (127.11.249.254:43933), 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427)
03:42:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:59.562880 12722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:42:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:59.566198 12722 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86decab305634c10acef3c81081e9320 (127.11.249.254:43933): Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111)
03:42:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:42:59.566756 12721 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427): Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:42:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:59.567023 12721 leader_election.cc:302] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 995018245ea348158f462c5199407a3c; no voters: 74d061b544644abab8983f3e0410fc16, 86decab305634c10acef3c81081e9320
03:42:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:42:59.567655 12824 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:43:00.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:00.229864 12700 master_runner.cc:186] Error getting master registration for 127.11.249.254:43933: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111),
03:43:00.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:00.230233 12700 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:43:00.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:00.231385 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:00.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:00.231606 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:01.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:01.234545 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:01.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:01.234794 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:02.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:02.238191 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:02.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:02.238448 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:02.252 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:43:02.254 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:43:02.254 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:02.255 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:02.255 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:02.256 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:02.257 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:02.257 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:02.259 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:43:03.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:03.241628 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:03.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:03.241882 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:03.912 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:43:03.913 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:43:03.915 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:03.915 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:03.915 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:03.915 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:03.916 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:03.916 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:03.919 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:43:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:04.245671 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:04.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:04.246024 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:04.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 27 03:43:04 dist-test-slave-4tf8 krb5kdc[12276](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790480584, etypes {rep=17 tkt=17 ses=17}, kudu/127.11.249.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:43:04.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:04.881999 12714 init.cc:295] Successfully reacquired a new kerberos TGT
03:43:05.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:05.233371 12700 master_runner.cc:186] Error getting master registration for 127.11.249.254:43933: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111),
03:43:05.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:05.233752 12700 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:43:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:05.248852 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:05.249115 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:06.251750 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:06.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:06.251979 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:07.254918 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:07.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:07.255220 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:07.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:43:07.773 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:43:07.773 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:07.774 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:07.774 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:07.776 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:07.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:07.776 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:07.778 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:43:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:08.258082 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:08.258319 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:09.261023 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:09.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:09.261266 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:10.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:10.236570 12700 master_runner.cc:186] Error getting master registration for 127.11.249.254:43933: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111),
03:43:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:10.236935 12700 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:43:10.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:10.264592 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:10.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:10.264840 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:10.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:10.495657 12841 raft_consensus.cc:482] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:43:10.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:10.496181 12841 raft_consensus.cc:504] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86decab305634c10acef3c81081e9320" member_type: VOTER last_known_addr { host: "127.11.249.254" port: 43933 } } peers { permanent_uuid: "74d061b544644abab8983f3e0410fc16" member_type: VOTER last_known_addr { host: "127.11.249.253" port: 43427 } } peers { permanent_uuid: "995018245ea348158f462c5199407a3c" member_type: VOTER last_known_addr { host: "127.11.249.252" port: 33077 } }
03:43:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:10.497768 12841 leader_election.cc:288] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86decab305634c10acef3c81081e9320 (127.11.249.254:43933), 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427)
03:43:10.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:10.498924 12722 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111) [suppressed 1 similar messages]
03:43:10.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:10.502036 12722 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 86decab305634c10acef3c81081e9320 (127.11.249.254:43933): Network error: Client connection negotiation failed: client connection to 127.11.249.254:43933: connect: Connection refused (error 111)
03:43:10.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:10.502562 12721 leader_election.cc:334] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 74d061b544644abab8983f3e0410fc16 (127.11.249.253:43427): Network error: Client connection negotiation failed: client connection to 127.11.249.253:43427: connect: Connection refused (error 111)
03:43:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:10.502838 12721 leader_election.cc:302] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 995018245ea348158f462c5199407a3c; no voters: 74d061b544644abab8983f3e0410fc16, 86decab305634c10acef3c81081e9320
03:43:10.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:10.503417 12841 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
03:43:11.112 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:43:11.113 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:43:11.113 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:11.113 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:11.114 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:11.115 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:11.115 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:11.116 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:11.118 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:43:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:11.268231 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:11.268513 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:12.271785 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:12.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:12.272069 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:12.791 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:43:12.792 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:43:12.793 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:12.793 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:12.793 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:12.793 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:12.794 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:12.794 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:12.796 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:43:13.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:13.275317 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:13.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:13.275586 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:14.152 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:43:14.153 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:43:14.153 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:14.154 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:14.154 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:14.155 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:14.155 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:14.155 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:14.158 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:43:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:14.278764 12787 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 995018245ea348158f462c5199407a3c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:43:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260927 03:43:14.279121 12787 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:43:14.651 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.254:43933(127.11.249.254:43933)] connecting to peer
03:43:14.652 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.11.249.253:43427(127.11.249.253:43427)] connecting to peer
03:43:14.653 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:14.654 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.11.249.254:43933(127.11.249.254:43933)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:14.654 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.254:43933: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933
03:43:14.654 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:14.654 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.253:43427(127.11.249.253:43427)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:14.654 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.11.249.253:43427: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
03:43:14.657 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077; will retry
03:43:14.662 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29627), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [196ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [557ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [581ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [586ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [589ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [598ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [604ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [618ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [621ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [659ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [663ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [759ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [771ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [800ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [803ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [805ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1055ms] refreshing cache from master, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1058ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1066ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1068ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1360ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1365ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1366ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1875ms] refreshing cache from master, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1878ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1883ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2175ms] refreshing cache from master, [2175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [2176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [2177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [2178ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [2180ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [2182ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [5016ms] refreshing cache from master, [5016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [5017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [5018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [5019ms] too many traces: truncated at 100 traces, deferred=Deferred@1372625260(state=PENDING, result=null, callback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918), errback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918)))
03:43:14.667 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.249.254:43933(127.11.249.254:43933)] disconnecting while in state TERMINATED
03:43:14.669 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.249.253:43427(127.11.249.253:43427)] disconnecting while in state TERMINATED
03:43:14.669 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.11.249.252:33077(127.11.249.252:33077)] disconnecting while in state READY
03:43:14.674 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:43:14.675 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.11.249.252:33077(127.11.249.252:33077)] handling channelInactive
03:43:14.676 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.11.249.252:33077(127.11.249.252:33077)] cleaning up while in state READY due to: connection closed
03:43:14.676 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:43:14.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:14.681048 12263 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 12700
03:43:15.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:43:15Z chronyd exiting
03:43:16.942 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:43:16.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:16.943519 11538 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 11820
03:43:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:16.970541 11538 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 11976
03:43:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:16.996891 11538 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 12121
03:43:17.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:17.023413 11538 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 11550
03:43:17.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:17.057157 11538 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 11620
03:43:17.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260927 03:43:17.087986 11538 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskGaJiXF/build/tsan/bin/kudu with pid 11695
03:43:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-27T03:43:18Z chronyd exiting
03:43:19.278 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29627), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [196ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [557ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [581ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [586ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [589ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [598ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [604ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [618ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [621ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [659ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [663ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [759ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [771ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [800ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [803ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [805ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1055ms] refreshing cache from master, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1058ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1066ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1068ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1360ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1365ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1366ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1875ms] refreshing cache from master, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1878ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1883ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2175ms] refreshing cache from master, [2175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [2176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [2177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [2178ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [2180ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [2182ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [5016ms] refreshing cache from master, [5016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [5017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [5018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [5019ms] too many traces: truncated at 100 traces, deferred=Deferred@1372625260(state=PENDING, result=null, callback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918), errback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.249.253:43427
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29627), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [196ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [557ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [581ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [586ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [589ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [598ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [604ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [618ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [621ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [659ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [663ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [759ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [771ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [800ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [803ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [805ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1055ms] refreshing cache from master, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1058ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1066ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1068ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1360ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1365ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1366ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1875ms] refreshing cache from master, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1878ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1883ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2175ms] refreshing cache from master, [2175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [2176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [2177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [2178ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [2180ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [2182ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [5016ms] refreshing cache from master, [5016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [5017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [5018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [5019ms] too many traces: truncated at 100 traces, deferred=Deferred@1372625260(state=PENDING, result=null, callback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918), errback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.249.253:43427
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
03:43:19.941 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
03:43:19.941 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29627), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [196ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [557ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [581ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [586ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [589ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [598ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [604ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [618ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [621ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [659ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [663ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [759ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [771ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [800ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [803ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [805ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1055ms] refreshing cache from master, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1058ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1066ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1068ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1360ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1365ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1366ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1875ms] refreshing cache from master, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1878ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1883ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2175ms] refreshing cache from master, [2175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [2176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [2177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [2178ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [2180ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [2182ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [5016ms] refreshing cache from master, [5016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [5017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [5018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [5019ms] too many traces: truncated at 100 traces, deferred=Deferred@1372625260(state=PENDING, result=null, callback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918), errback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.249.253:43427
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29627), Traces: [0ms] refreshing cache from master, [46ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [128ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [129ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [196ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [557ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [561ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [581ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [586ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [588ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [589ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [595ms] refreshing cache from master, [595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [598ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [604ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [605ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [606ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [618ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [621ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [623ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [625ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [659ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [661ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [663ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [695ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [699ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [700ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [704ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [759ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [769ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [770ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [771ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [795ms] refreshing cache from master, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [800ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [803ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [805ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [806ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1055ms] refreshing cache from master, [1056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1058ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1060ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1066ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1067ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1068ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1355ms] refreshing cache from master, [1356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1359ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1360ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1361ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1365ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1366ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1875ms] refreshing cache from master, [1876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [1877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [1878ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [1879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [1881ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [1883ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [1883ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2175ms] refreshing cache from master, [2175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [2176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [2177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [2178ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.254:43933: Network error: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933, [2180ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.253:43427: Network error: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [2182ms] Sub RPC ConnectToMaster: received response from server master-127.11.249.252:33077: OK, [2182ms] delaying RPC due to: Service unavailable: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427, [5016ms] refreshing cache from master, [5016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.254:43933, [5017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.253:43427, [5018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.11.249.252:33077, [5019ms] too many traces: truncated at 100 traces, deferred=Deferred@1372625260(state=PENDING, result=null, callback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918), errback=(continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.11.249.254:43933,127.11.249.253:43427,127.11.249.252:33077) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.254:43933(127.11.249.254:43933): Connection refused: /127.11.249.254:43933,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.11.249.253:43427(127.11.249.253:43427): Connection refused: /127.11.249.253:43427
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.11.249.253:43427
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)