07:04:35.829 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:04:35.841 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskzJbp2h/test-tmp
07:04:35.844 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:04:35.848 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskzJbp2h/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:04:36.069 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:04:36.260 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:04:36.261 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:04:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.266320 374 mini_oidc.cc:152] Starting JWKS server
07:04:36.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.271224 374 webserver.cc:460] Webserver started at http://127.0.0.1:45181/ using document root <none> and password file <none>
07:04:36.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.271733 374 mini_oidc.cc:189] Starting OIDC Discovery server
07:04:36.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.273550 374 webserver.cc:460] Webserver started at http://127.0.0.1:33491/ using document root <none> and password file <none>
07:04:36.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T07:04:36Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:04:36.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T07:04:36Z Disabled control of system clock
07:04:36.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.324394 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:36.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:36.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/wal
07:04:36.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/data
07:04:36.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:36.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:36.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:36.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:36.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:36.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:36.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/logs
07:04:36.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/data/info.pb
07:04:36.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:36.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:36.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:41871
07:04:36.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
07:04:36.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:36.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:36.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:36.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:36.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45181/jwks.json with env {}
07:04:36.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.656306 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:36.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.656865 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:36.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.657488 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:36.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.688810 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:04:36.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.689118 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:36.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.689314 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:04:36.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.689566 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:04:36.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.689847 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:04:36.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.690061 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45181/jwks.json
07:04:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.725829 386 master_runner.cc:386] Master server non-default flags:
07:04:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:36.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/data
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/wal
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:41871
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:36.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45181/jwks.json
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/data/info.pb
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:36.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/logs
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:36.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:36.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.727248 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:36.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.729363 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:36.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.751075 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.751279 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:36.752771 386 server_base.cc:1034] running on GCE node
07:04:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:36.751619 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:37.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:37.993901 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:37.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:37.996704 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:37.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:37.998136 386 hybrid_clock.cc:648] HybridClock initialized: now 1790665477998058 us; error 83 us; skew 500 ppm
07:04:37.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:37.999161 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:38.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.009855 386 webserver.cc:460] Webserver started at http://127.0.93.190:45207/ using document root <none> and password file <none>
07:04:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.011117 386 fs_manager.cc:359] Metadata directory not provided
07:04:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.011399 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.011994 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.017592 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/data/instance:
07:04:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "838638ab6fe84c2c9e6d76785afe833f"
07:04:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:38 on dist-test-slave-tfv6"
07:04:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.019275 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/wal/instance:
07:04:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "838638ab6fe84c2c9e6d76785afe833f"
07:04:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:38 on dist-test-slave-tfv6"
07:04:38.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.028051 386 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
07:04:38.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.034119 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:38.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.035329 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
07:04:38.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.035701 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/wal
07:04:38.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "838638ab6fe84c2c9e6d76785afe833f"
07:04:38.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:38 on dist-test-slave-tfv6"
07:04:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.036132 386 fs_report.cc:383] FS layout report
07:04:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/wal
07:04:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/wal
07:04:38.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/data/data
07:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:38.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:38.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:38.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:38.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:38.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.076406 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:38.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.078086 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:38.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.078655 386 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.159850 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:41871
07:04:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.159955 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:41871 every 8 connection(s)
07:04:38.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.162745 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/data/info.pb
07:04:38.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.168949 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:04:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.173167 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 386
07:04:38.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.173591 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-0/wal/instance
07:04:38.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.177692 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:38.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:38.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/wal
07:04:38.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/data
07:04:38.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/logs
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/data/info.pb
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:38.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:40317
07:04:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
07:04:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45181/jwks.json with env {}
07:04:38.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.184089 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } has no permanent_uuid. Determining permanent_uuid...
07:04:38.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.218466 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } has no permanent_uuid. Determining permanent_uuid...
07:04:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.220927 406 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40317: connect: Connection refused (error 111)
07:04:38.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.226248 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40317: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40317: connect: Connection refused (error 111)
07:04:38.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.275900 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } attempt: 1
07:04:38.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.281325 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40317: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40317: connect: Connection refused (error 111)
07:04:38.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.349831 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } attempt: 2
07:04:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.353866 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40317: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40317: connect: Connection refused (error 111)
07:04:38.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.445392 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } attempt: 3
07:04:38.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.449921 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40317: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40317: connect: Connection refused (error 111)
07:04:38.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.517104 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.517721 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:38.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.518273 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.555682 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:04:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.555970 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.556306 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:04:38.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.556605 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:04:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.556838 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:04:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.557021 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45181/jwks.json
07:04:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.593483 456 master_runner.cc:386] Master server non-default flags:
07:04:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/data
07:04:38.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/wal
07:04:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:40317
07:04:38.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45181/jwks.json
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/data/info.pb
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/logs
07:04:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:38.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.593436 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } attempt: 4
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.594934 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.597026 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:38.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.598649 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40317: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40317: connect: Connection refused (error 111)
07:04:38.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.616878 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.898049 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } attempt: 5
07:04:38.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.901681 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40317: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40317: connect: Connection refused (error 111)
07:04:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.449409 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } attempt: 6
07:04:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:39.457732 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:40317: Network error: Client connection negotiation failed: client connection to 127.0.93.189:40317: connect: Connection refused (error 111)
07:04:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.616894 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:38.617028 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:39.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:38.618794 456 server_base.cc:1034] running on GCE node
07:04:39.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.845477 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:39.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.848198 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:39.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.849589 456 hybrid_clock.cc:648] HybridClock initialized: now 1790665479849539 us; error 66 us; skew 500 ppm
07:04:39.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.850796 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:39.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.858708 456 webserver.cc:460] Webserver started at http://127.0.93.189:41661/ using document root <none> and password file <none>
07:04:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.859899 456 fs_manager.cc:359] Metadata directory not provided
07:04:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.860173 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:39.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.860733 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:39.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.865856 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/data/instance:
07:04:39.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8dc6eda494f64e938baa51411fd4e6a7"
07:04:39.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:39 on dist-test-slave-tfv6"
07:04:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.867228 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/wal/instance:
07:04:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8dc6eda494f64e938baa51411fd4e6a7"
07:04:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:39 on dist-test-slave-tfv6"
07:04:39.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.874738 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
07:04:39.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.880788 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:39.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.881974 456 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
07:04:39.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.882274 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/wal
07:04:39.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8dc6eda494f64e938baa51411fd4e6a7"
07:04:39.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:39 on dist-test-slave-tfv6"
07:04:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.882697 456 fs_report.cc:383] FS layout report
07:04:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/wal
07:04:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/wal
07:04:39.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/data/data
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.933208 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.934624 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:39.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:39.935034 456 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:40.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.013134 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:40317
07:04:40.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.013443 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:40317 every 8 connection(s)
07:04:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.016324 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/data/info.pb
07:04:40.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.020270 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 456
07:04:40.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.020754 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-1/wal/instance
07:04:40.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.022673 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.023748 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/wal
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/data
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/logs
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/data/info.pb
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:46823
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45181/jwks.json with env {}
07:04:40.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.037359 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } has no permanent_uuid. Determining permanent_uuid...
07:04:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.062220 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } has no permanent_uuid. Determining permanent_uuid...
07:04:40.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.079087 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } has no permanent_uuid. Determining permanent_uuid...
07:04:40.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.081023 481 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.085296 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.134785 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 1
07:04:40.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.138841 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.207288 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 2
07:04:40.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.212579 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.304037 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 3
07:04:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.308054 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.351687 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.352221 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.352679 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:40.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.384500 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:04:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.384786 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.384972 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:04:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.385145 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:04:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.385330 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:04:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.385469 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45181/jwks.json
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.424651 531 master_runner.cc:386] Master server non-default flags:
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/data
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/wal
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:46823
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45181/jwks.json
07:04:40.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/data/info.pb
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/logs
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:40.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:04:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.425988 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:40.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.428273 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.450942 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:40.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.451507 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 4
07:04:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.456818 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.518362 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } attempt: 7
07:04:40.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.555266 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } has no permanent_uuid. Determining permanent_uuid...
07:04:40.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.574196 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.632886 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 1
07:04:40.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.642441 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.724115 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 2
07:04:40.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.730698 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.756307 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 5
07:04:40.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.763303 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.816284 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 3
07:04:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.824294 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:40.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:40.964888 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 4
07:04:40.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.971603 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.255180 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 5
07:04:41.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:41.261212 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:41.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.310958 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 6
07:04:41.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:41.321835 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:41.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.450995 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:41.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:40.453730 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:41.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:41.734328 543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1282 milliseconds
07:04:41.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.734462 531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:04:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.735821 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:41.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.738991 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:41.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.740428 531 hybrid_clock.cc:648] HybridClock initialized: now 1790665481740384 us; error 62 us; skew 500 ppm
07:04:41.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.741246 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:41.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.748214 531 webserver.cc:460] Webserver started at http://127.0.93.188:40531/ using document root <none> and password file <none>
07:04:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.749102 531 fs_manager.cc:359] Metadata directory not provided
07:04:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.749289 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.749682 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:41.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.754519 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/data/instance:
07:04:41.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff5bf1afc1c6482293e95c0dbba87089"
07:04:41.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:41 on dist-test-slave-tfv6"
07:04:41.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.755640 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/wal/instance:
07:04:41.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff5bf1afc1c6482293e95c0dbba87089"
07:04:41.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:41 on dist-test-slave-tfv6"
07:04:41.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.763648 531 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
07:04:41.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.769696 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:41.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.770838 531 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:04:41.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.771214 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/wal
07:04:41.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ff5bf1afc1c6482293e95c0dbba87089"
07:04:41.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:41 on dist-test-slave-tfv6"
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.771613 531 fs_report.cc:383] FS layout report
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/wal
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/wal
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/data/data
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:41.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:41.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:41.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.813751 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 6
07:04:41.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:41.820655 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:46823: Network error: Client connection negotiation failed: client connection to 127.0.93.188:46823: connect: Connection refused (error 111)
07:04:41.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.827814 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:41.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.829274 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:41.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.829676 531 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:41.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.911737 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:46823
07:04:41.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.911828 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:46823 every 8 connection(s)
07:04:41.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.914434 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/data/info.pb
07:04:41.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.915612 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 531
07:04:41.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.916131 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/master-2/wal/instance
07:04:41.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.921350 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:04:41.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.942939 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } has no permanent_uuid. Determining permanent_uuid...
07:04:41.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.970861 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } has no permanent_uuid. Determining permanent_uuid...
07:04:41.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:41.982197 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } has no permanent_uuid. Determining permanent_uuid...
07:04:42.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.001398 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089: Bootstrap starting.
07:04:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.007755 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089: Neither blocks nor log segments found. Creating new log.
07:04:42.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.009829 609 log.cc:824] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089: Log is configured to *not* fsync() on all Append() calls
07:04:42.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.014953 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089: No bootstrap required, opened a new log
07:04:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.037169 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.037807 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:04:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.038074 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ff5bf1afc1c6482293e95c0dbba87089, State: Initialized, Role: FOLLOWER
07:04:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.038967 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.042789 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:42.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.043592 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: This master's current role is: FOLLOWER
07:04:42.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.044602 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: configured and running, proceeding with master startup.
07:04:42.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:42.072286 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:04:42.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:42.073110 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:04:42.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:42.086419 563 tablet.cc:2369] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.067687 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:41871: OK, instance_id { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" instance_seqno: 1790665478137701 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:04:42.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.104282 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:04:42.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.253818 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:04:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.254482 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.257229 618 leader_election.cc:288] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 838638ab6fe84c2c9e6d76785afe833f (127.0.93.190:41871), 8dc6eda494f64e938baa51411fd4e6a7 (127.0.93.189:40317)
07:04:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.382426 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 7
07:04:42.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.407521 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7: Bootstrap starting.
07:04:42.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.415927 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7: Neither blocks nor log segments found. Creating new log.
07:04:42.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.418351 529 log.cc:824] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7: Log is configured to *not* fsync() on all Append() calls
07:04:42.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.427318 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7: No bootstrap required, opened a new log
07:04:42.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.460163 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.461066 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:04:42.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.461341 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8dc6eda494f64e938baa51411fd4e6a7, State: Initialized, Role: FOLLOWER
07:04:42.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.462276 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.465608 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:42.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.466856 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [sys.catalog]: This master's current role is: FOLLOWER
07:04:42.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.467453 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [sys.catalog]: configured and running, proceeding with master startup.
07:04:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.467654 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ff5bf1afc1c6482293e95c0dbba87089" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8dc6eda494f64e938baa51411fd4e6a7" is_pre_election: true
07:04:42.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.468537 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ff5bf1afc1c6482293e95c0dbba87089 in term 0.
07:04:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.470366 559 leader_election.cc:302] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8dc6eda494f64e938baa51411fd4e6a7, ff5bf1afc1c6482293e95c0dbba87089; no voters:
07:04:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.471438 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:04:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.471776 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:04:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.472126 618 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 0 FOLLOWER]: Advancing to term 1
07:04:42.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.481153 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.483316 618 leader_election.cc:288] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [CANDIDATE]: Term 1 election: Requested vote from peers 838638ab6fe84c2c9e6d76785afe833f (127.0.93.190:41871), 8dc6eda494f64e938baa51411fd4e6a7 (127.0.93.189:40317)
07:04:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.491746 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ff5bf1afc1c6482293e95c0dbba87089" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8dc6eda494f64e938baa51411fd4e6a7"
07:04:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.492333 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [term 0 FOLLOWER]: Advancing to term 1
07:04:42.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.498945 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ff5bf1afc1c6482293e95c0dbba87089 in term 1.
07:04:42.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.500538 559 leader_election.cc:302] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8dc6eda494f64e938baa51411fd4e6a7, ff5bf1afc1c6482293e95c0dbba87089; no voters:
07:04:42.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.501784 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 1 FOLLOWER]: Leader election won for term 1
07:04:42.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.518798 618 raft_consensus.cc:686] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [term 1 LEADER]: Becoming Leader. State: Replica: ff5bf1afc1c6482293e95c0dbba87089, State: Running, Role: LEADER
07:04:42.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.521494 618 consensus_queue.cc:237] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.530511 645 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:04:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.534577 642 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ff5bf1afc1c6482293e95c0dbba87089. Latest consensus state: current_term: 1 leader_uuid: "ff5bf1afc1c6482293e95c0dbba87089" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:42.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.535166 642 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: This master's current role is: LEADER
07:04:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.539526 647 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:04:42.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.546747 647 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:04:42.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.537487 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:41871: OK, instance_id { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" instance_seqno: 1790665478137701 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:04:42.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.572727 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:04:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.580354 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [term 1 FOLLOWER]: Refusing update from remote peer ff5bf1afc1c6482293e95c0dbba87089: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:04:42.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.582003 618 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:04:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.604346 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ff5bf1afc1c6482293e95c0dbba87089. Latest consensus state: current_term: 1 leader_uuid: "ff5bf1afc1c6482293e95c0dbba87089" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:42.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.605465 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [sys.catalog]: This master's current role is: FOLLOWER
07:04:42.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.605908 648 mvcc.cc:205] Tried to move back new op lower bound from 7334565816633442304 to 7334565816421318656. Current Snapshot: MvccSnapshot[applied={T|T < 7334565816633442304}]
07:04:42.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.607389 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ff5bf1afc1c6482293e95c0dbba87089" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:42.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.608294 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: This master's current role is: LEADER
07:04:42.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.615913 649 mvcc.cc:205] Tried to move back new op lower bound from 7334565816633442304 to 7334565816421318656. Current Snapshot: MvccSnapshot[applied={T|T < 7334565816633442304}]
07:04:42.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.617463 647 catalog_manager.cc:1347] Generated new cluster ID: b5a36f2f0f18463e8ea658a9dd82f35c
07:04:42.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.617754 647 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:04:42.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.620891 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ff5bf1afc1c6482293e95c0dbba87089" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:42.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.621670 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7 [sys.catalog]: This master's current role is: FOLLOWER
07:04:42.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.662930 647 catalog_manager.cc:1370] Generated new certificate authority record
07:04:42.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.665946 647 catalog_manager.cc:1504] Loading token signing keys...
07:04:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.698952 647 catalog_manager.cc:5868] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089: Generated new TSK 0
07:04:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.700774 647 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:04:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.732143 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/wal
07:04:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/data
07:04:42.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/logs
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/data/info.pb
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:42.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:04:42.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.854101 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } attempt: 7
07:04:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.894407 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f: Bootstrap starting.
07:04:42.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.903925 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f: Neither blocks nor log segments found. Creating new log.
07:04:42.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.906898 454 log.cc:824] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f: Log is configured to *not* fsync() on all Append() calls
07:04:42.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.919473 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f: No bootstrap required, opened a new log
07:04:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.960187 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.961686 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:04:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.962241 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 838638ab6fe84c2c9e6d76785afe833f, State: Initialized, Role: FOLLOWER
07:04:42.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.963598 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } }
07:04:42.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.977202 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [sys.catalog]: configured and running, proceeding with master startup.
07:04:42.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.977414 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ff5bf1afc1c6482293e95c0dbba87089" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "838638ab6fe84c2c9e6d76785afe833f" is_pre_election: true
07:04:42.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.978171 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ff5bf1afc1c6482293e95c0dbba87089" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "838638ab6fe84c2c9e6d76785afe833f"
07:04:42.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.979380 427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [term 0 FOLLOWER]: Advancing to term 1
07:04:42.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:42.995412 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [term 1 FOLLOWER]: Refusing update from remote peer ff5bf1afc1c6482293e95c0dbba87089: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:04:43.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.002692 642 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [LEADER]: Connected to new peer: Peer: permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
07:04:43.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.006727 656 sys_catalog.cc:455] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:43.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.008435 656 sys_catalog.cc:458] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [sys.catalog]: This master's current role is: FOLLOWER
07:04:43.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.065973 656 sys_catalog.cc:455] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [sys.catalog]: SysCatalogTable state changed. Reason: New leader ff5bf1afc1c6482293e95c0dbba87089. Latest consensus state: current_term: 1 leader_uuid: "ff5bf1afc1c6482293e95c0dbba87089" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:43.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.065871 659 sys_catalog.cc:455] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ff5bf1afc1c6482293e95c0dbba87089" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:43.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.066728 659 sys_catalog.cc:458] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [sys.catalog]: This master's current role is: FOLLOWER
07:04:43.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.066696 656 sys_catalog.cc:458] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f [sys.catalog]: This master's current role is: FOLLOWER
07:04:43.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.121299 672 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:04:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:43.228576 653 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:43.229171 653 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:43.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:43.229858 653 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:43.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:43.262395 653 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:43.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:43.263455 653 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.299562 653 tablet_server_runner.cc:78] Tablet server non-default flags:
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/data
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/wal
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/data/info.pb
07:04:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
07:04:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:43.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/logs
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:43.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.301103 653 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:43.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.303220 653 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:43.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:43.325886 679 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:43.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.457198 642 sys_catalog.cc:455] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ff5bf1afc1c6482293e95c0dbba87089" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "838638ab6fe84c2c9e6d76785afe833f" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 41871 } } peers { permanent_uuid: "8dc6eda494f64e938baa51411fd4e6a7" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 40317 } } peers { permanent_uuid: "ff5bf1afc1c6482293e95c0dbba87089" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 46823 } } }
07:04:43.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.460638 642 sys_catalog.cc:458] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 [sys.catalog]: This master's current role is: LEADER
07:04:43.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.539454 645 catalog_manager.cc:1259] Loaded cluster ID: b5a36f2f0f18463e8ea658a9dd82f35c
07:04:43.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.540166 645 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7: loading cluster ID for follower catalog manager: success
07:04:43.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.562610 645 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7: acquiring CA information for follower catalog manager: success
07:04:43.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.571938 645 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8dc6eda494f64e938baa51411fd4e6a7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:04:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.130065 672 catalog_manager.cc:1259] Loaded cluster ID: b5a36f2f0f18463e8ea658a9dd82f35c
07:04:44.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.130729 672 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f: loading cluster ID for follower catalog manager: success
07:04:44.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.151340 672 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f: acquiring CA information for follower catalog manager: success
07:04:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.164222 672 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 838638ab6fe84c2c9e6d76785afe833f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:04:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:43.328270 653 server_base.cc:1034] running on GCE node
07:04:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:43.326788 678 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:44.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:43.331135 681 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:44.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.822263 653 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:44.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.830859 653 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:44.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.834815 653 hybrid_clock.cc:648] HybridClock initialized: now 1790665484834735 us; error 66 us; skew 500 ppm
07:04:44.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.836222 653 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:44.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.862743 653 webserver.cc:460] Webserver started at http://127.0.93.129:46849/ using document root <none> and password file <none>
07:04:44.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.864384 653 fs_manager.cc:359] Metadata directory not provided
07:04:44.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.864909 653 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:44.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.865614 653 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.873518 653 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/data/instance:
07:04:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e21312b736614f99bc593b8d528fd174"
07:04:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:44 on dist-test-slave-tfv6"
07:04:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.879936 653 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/wal/instance:
07:04:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e21312b736614f99bc593b8d528fd174"
07:04:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:44 on dist-test-slave-tfv6"
07:04:44.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.896512 653 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.009s sys 0.004s
07:04:44.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.910717 694 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:44.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.912397 653 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.010s sys 0.001s
07:04:44.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.912998 653 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/wal
07:04:44.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e21312b736614f99bc593b8d528fd174"
07:04:44.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:44 on dist-test-slave-tfv6"
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.913661 653 fs_report.cc:383] FS layout report
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/wal
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/wal
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/data/data
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:44.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:44.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:44.998919 653 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:45.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.001235 653 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:45.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.002125 653 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:45.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.011940 653 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:04:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.024616 653 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:04:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.025214 653 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
07:04:45.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.025717 653 ts_tablet_manager.cc:610] Registered 0 tablets
07:04:45.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.026062 653 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:04:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.363306 653 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:44125
07:04:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.363409 806 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:44125 every 8 connection(s)
07:04:45.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.366235 653 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/data/info.pb
07:04:45.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.368614 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 653
07:04:45.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.369185 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-0/wal/instance
07:04:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.390239 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/wal
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/data
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/logs
07:04:45.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/data/info.pb
07:04:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:45.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:04:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.478421 808 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40317
07:04:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.478945 808 heartbeater.cc:464] Registering TS with master...
07:04:45.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.480243 808 heartbeater.cc:511] Master 127.0.93.189:40317 requested a full tablet report, sending...
07:04:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.484922 494 ts_manager.cc:194] Registered new tserver with Master: e21312b736614f99bc593b8d528fd174 (127.0.93.129:44125)
07:04:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.487270 807 heartbeater.cc:347] Connected to a master server at 127.0.93.188:46823
07:04:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.487661 807 heartbeater.cc:464] Registering TS with master...
07:04:45.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.488547 807 heartbeater.cc:511] Master 127.0.93.188:46823 requested a full tablet report, sending...
07:04:45.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.491902 574 ts_manager.cc:194] Registered new tserver with Master: e21312b736614f99bc593b8d528fd174 (127.0.93.129:44125)
07:04:45.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.495085 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:34365
07:04:45.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.498960 809 heartbeater.cc:347] Connected to a master server at 127.0.93.190:41871
07:04:45.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.499320 809 heartbeater.cc:464] Registering TS with master...
07:04:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.501591 809 heartbeater.cc:511] Master 127.0.93.190:41871 requested a full tablet report, sending...
07:04:45.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.505601 419 ts_manager.cc:194] Registered new tserver with Master: e21312b736614f99bc593b8d528fd174 (127.0.93.129:44125)
07:04:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:45.804509 812 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:45.805143 812 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:45.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:45.805759 812 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:45.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:45.838480 812 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:45.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:45.839600 812 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.877986 812 tablet_server_runner.cc:78] Tablet server non-default flags:
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/data
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/wal
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:45.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/data/info.pb
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/logs
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:45.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:45.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.879545 812 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:45.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:45.881687 812 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:45.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:45.914211 823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:46.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:46.501766 807 heartbeater.cc:503] Master 127.0.93.188:46823 was elected leader, sending a full tablet report...
07:04:47.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.168987 525 debug-util.cc:398] Leaking SignalData structure 0x7b0800083fc0 after lost signal to thread 460
07:04:47.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.169832 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba780 after lost signal to thread 528
07:04:47.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.173144 803 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 673
07:04:47.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.174585 803 debug-util.cc:398] Leaking SignalData structure 0x7b08000aff40 after lost signal to thread 806
07:04:47.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.304343 822 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 812
07:04:47.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.646502 822 kernel_stack_watchdog.cc:198] Thread 812 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 383ms:
07:04:47.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
07:04:47.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
07:04:47.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
07:04:47.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
07:04:47.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.646852 825 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1727 milliseconds
07:04:47.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.648063 826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:47.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.650548 812 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.732s user 0.314s sys 0.655s
07:04:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:47.650889 812 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.732s user 0.314s sys 0.655s
07:04:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.651144 812 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:04:47.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:45.931265 824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:47.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.654422 812 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:47.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.669122 812 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:47.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.670639 812 hybrid_clock.cc:648] HybridClock initialized: now 1790665487670602 us; error 55 us; skew 500 ppm
07:04:47.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.671777 812 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:47.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.700192 812 webserver.cc:460] Webserver started at http://127.0.93.130:42509/ using document root <none> and password file <none>
07:04:47.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.701704 812 fs_manager.cc:359] Metadata directory not provided
07:04:47.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.702016 812 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:47.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.702816 812 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:47.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.711051 812 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/data/instance:
07:04:47.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30230e8a008d4e298bc96307945c2d43"
07:04:47.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:47 on dist-test-slave-tfv6"
07:04:47.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.712702 812 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/wal/instance:
07:04:47.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30230e8a008d4e298bc96307945c2d43"
07:04:47.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:47 on dist-test-slave-tfv6"
07:04:47.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.728760 812 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.011s sys 0.003s
07:04:47.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.744035 835 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:47.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.745585 812 fs_manager.cc:725] Time spent opening block manager: real 0.012s user 0.000s sys 0.007s
07:04:47.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.746025 812 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/wal
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "30230e8a008d4e298bc96307945c2d43"
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:47 on dist-test-slave-tfv6"
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.746531 812 fs_report.cc:383] FS layout report
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/wal
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/wal
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/data/data
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:47.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.830188 812 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:47.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.832345 812 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:47.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.832989 812 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:47.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.842625 812 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:04:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.868242 812 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:04:47.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.868626 812 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:04:47.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.868990 812 ts_tablet_manager.cc:610] Registered 0 tablets
07:04:47.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:47.869261 812 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:04:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.355680 812 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:40949
07:04:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.355806 947 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:40949 every 8 connection(s)
07:04:48.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.359448 812 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/data/info.pb
07:04:48.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.364831 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 812
07:04:48.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.365728 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-1/wal/instance
07:04:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.398022 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/wal
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/data
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/logs
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/data/info.pb
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:04:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
07:04:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
07:04:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
07:04:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:48.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:48.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:48.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:04:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.505663 950 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40317
07:04:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.506668 950 heartbeater.cc:464] Registering TS with master...
07:04:48.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.507973 950 heartbeater.cc:511] Master 127.0.93.189:40317 requested a full tablet report, sending...
07:04:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.515120 953 heartbeater.cc:347] Connected to a master server at 127.0.93.190:41871
07:04:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.515537 953 heartbeater.cc:464] Registering TS with master...
07:04:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.516659 494 ts_manager.cc:194] Registered new tserver with Master: 30230e8a008d4e298bc96307945c2d43 (127.0.93.130:40949)
07:04:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.516898 953 heartbeater.cc:511] Master 127.0.93.190:41871 requested a full tablet report, sending...
07:04:48.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.520203 419 ts_manager.cc:194] Registered new tserver with Master: 30230e8a008d4e298bc96307945c2d43 (127.0.93.130:40949)
07:04:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.538543 948 heartbeater.cc:347] Connected to a master server at 127.0.93.188:46823
07:04:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.538965 948 heartbeater.cc:464] Registering TS with master...
07:04:48.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.539914 948 heartbeater.cc:511] Master 127.0.93.188:46823 requested a full tablet report, sending...
07:04:48.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.542951 574 ts_manager.cc:194] Registered new tserver with Master: 30230e8a008d4e298bc96307945c2d43 (127.0.93.130:40949)
07:04:48.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.545781 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:32857
07:04:48.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:48.823406 949 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:48.823961 949 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:48.824589 949 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:48.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:48.877701 949 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:48.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:48.879101 949 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
07:04:48.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.959297 949 tablet_server_runner.cc:78] Tablet server non-default flags:
07:04:48.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:48.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:37885
07:04:48.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/data
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/wal
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/data/info.pb
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:41871,127.0.93.189:40317,127.0.93.188:46823
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/logs
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:48.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:48.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.962270 949 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:48.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:48.964646 949 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:48.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:48.989741 965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:49.551410 948 heartbeater.cc:503] Master 127.0.93.188:46823 was elected leader, sending a full tablet report...
07:04:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:49.002795 968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:50.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:49.000211 966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:50.470155 967 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:04:50.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.470285 949 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:04:50.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.474193 949 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:50.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.476670 949 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:50.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.478186 949 hybrid_clock.cc:648] HybridClock initialized: now 1790665490478062 us; error 131 us; skew 500 ppm
07:04:50.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.479154 949 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:50.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.492057 949 webserver.cc:460] Webserver started at http://127.0.93.131:43457/ using document root <none> and password file <none>
07:04:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.493072 949 fs_manager.cc:359] Metadata directory not provided
07:04:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.493300 949 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:50.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.493739 949 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.499037 949 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/data/instance:
07:04:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd22461c27164a89bc2d43690a18cf56"
07:04:50.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:50 on dist-test-slave-tfv6"
07:04:50.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.500241 949 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/wal/instance:
07:04:50.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd22461c27164a89bc2d43690a18cf56"
07:04:50.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:50 on dist-test-slave-tfv6"
07:04:50.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.508397 949 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
07:04:50.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.515094 976 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:50.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.516389 949 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:04:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.516790 949 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/wal
07:04:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fd22461c27164a89bc2d43690a18cf56"
07:04:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:50 on dist-test-slave-tfv6"
07:04:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.517266 949 fs_report.cc:383] FS layout report
07:04:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/wal
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/wal
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/data/data
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:50.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:50.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.573341 949 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.575114 949 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:50.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.575687 949 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:50.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.578416 949 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:04:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.582994 949 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:04:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.583288 949 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:04:50.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.583639 949 ts_tablet_manager.cc:610] Registered 0 tablets
07:04:50.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.583858 949 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:04:50.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.744405 949 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:36615
07:04:50.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.744511 1088 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:36615 every 8 connection(s)
07:04:50.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.747424 949 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/data/info.pb
07:04:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.756119 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 949
07:04:50.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.756693 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster7991857348120918594/ts-2/wal/instance
07:04:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.799417 1089 heartbeater.cc:347] Connected to a master server at 127.0.93.188:46823
07:04:50.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.799860 1089 heartbeater.cc:464] Registering TS with master...
07:04:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.801263 1089 heartbeater.cc:511] Master 127.0.93.188:46823 requested a full tablet report, sending...
07:04:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.805001 574 ts_manager.cc:194] Registered new tserver with Master: fd22461c27164a89bc2d43690a18cf56 (127.0.93.131:36615)
07:04:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.807515 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:49669
07:04:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.810438 1090 heartbeater.cc:347] Connected to a master server at 127.0.93.189:40317
07:04:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.810832 1090 heartbeater.cc:464] Registering TS with master...
07:04:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.812392 1090 heartbeater.cc:511] Master 127.0.93.189:40317 requested a full tablet report, sending...
07:04:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.814222 1091 heartbeater.cc:347] Connected to a master server at 127.0.93.190:41871
07:04:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.814618 1091 heartbeater.cc:464] Registering TS with master...
07:04:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.815141 1091 heartbeater.cc:511] Master 127.0.93.190:41871 requested a full tablet report, sending...
07:04:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.815979 494 ts_manager.cc:194] Registered new tserver with Master: fd22461c27164a89bc2d43690a18cf56 (127.0.93.131:36615)
07:04:50.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.817977 418 ts_manager.cc:194] Registered new tserver with Master: fd22461c27164a89bc2d43690a18cf56 (127.0.93.131:36615)
07:04:50.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:50.835826 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:04:50.847 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:04:50.848 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:04:50.859 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 41871
}
bound_http_address {
host: "127.0.93.190"
port: 45207
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 40317
}
bound_http_address {
host: "127.0.93.189"
port: 41661
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 46823
}
bound_http_address {
host: "127.0.93.188"
port: 40531
}
}
}
07:04:50.863 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:04:50.866 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 44125
}
bound_http_address {
host: "127.0.93.129"
port: 46849
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 40949
}
bound_http_address {
host: "127.0.93.130"
port: 42509
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 36615
}
bound_http_address {
host: "127.0.93.131"
port: 43457
}
}
}
07:04:50.867 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:04:51.067 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:04:51.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:51.177621 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba900 after lost signal to thread 460
07:04:51.191 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskzJbp2h/test-tmp
07:04:51.192 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:04:51.192 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskzJbp2h/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:04:51.212 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:04:51.606 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:04:51.607 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:04:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:51.610153 1100 mini_oidc.cc:152] Starting JWKS server
07:04:51.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:51.613988 1100 webserver.cc:460] Webserver started at http://127.0.0.1:44975/ using document root <none> and password file <none>
07:04:51.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:51.614486 1100 mini_oidc.cc:189] Starting OIDC Discovery server
07:04:51.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:51.616215 1100 webserver.cc:460] Webserver started at http://127.0.0.1:33885/ using document root <none> and password file <none>
07:04:51.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:04:51.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/principal' for realm 'KRBTEST.COM',
07:04:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:04:51.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:51 dist-test-slave-tfv6 krb5kdc[1112](info): setting up network...
07:04:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:04:51.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:51 dist-test-slave-tfv6 krb5kdc[1112](info): set up 2 sockets
07:04:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:51 dist-test-slave-tfv6 krb5kdc[1112](info): commencing operation
07:04:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:04:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:51.817309 1089 heartbeater.cc:503] Master 127.0.93.188:46823 was elected leader, sending a full tablet report...
07:04:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:52.773644 1100 mini_kdc.cc:121] Time spent starting KDC: real 1.131s user 0.002s sys 0.019s
07:04:52.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:04:52.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:04:52.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:04:52.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:04:52.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:04:52.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:04:52.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:04:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:04:52.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:04:52.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:04:52.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/test-user.keytab.
07:04:52.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/test-user.keytab.
07:04:52.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:52 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665492, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:04:52.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:04:52.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T07:04:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:04:52.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T07:04:52Z Disabled control of system clock
07:04:52.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.62@KRBTEST.COM; defaulting to no policy
07:04:52.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:04:52.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.62@KRBTEST.COM" created.
07:04:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:04:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.62.keytab.
07:04:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.62.keytab.
07:04:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:53.027122 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:04:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/logs
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/info.pb
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:04:53.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:43129
07:04:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
07:04:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.62.keytab
07:04:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:53.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:04:53.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:04:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.378067 1125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:53.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.378715 1125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:53.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.379290 1125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.411037 1125 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:04:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.411263 1125 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:04:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.411528 1125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.411724 1125 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:04:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.411916 1125 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:04:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.412094 1125 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:04:53.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.412233 1125 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44975/jwks.json
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:53.457958 1125 master_runner.cc:386] Master server non-default flags:
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.62.keytab
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:43129
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/info.pb
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/logs
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:53.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:53.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:53.459229 1125 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:53.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:53.460201 1125 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:53.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.479916 1135 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:53.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.894457 1085 debug-util.cc:398] Leaking SignalData structure 0x7b08000af280 after lost signal to thread 960
07:04:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.895675 1085 debug-util.cc:398] Leaking SignalData structure 0x7b08000afea0 after lost signal to thread 1088
07:04:54.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.479964 1132 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:54.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:53.481824 1125 server_base.cc:1034] running on GCE node
07:04:54.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:53.481828 1133 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:54.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.792258 1125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:54.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.796294 1125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:54.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.797971 1125 hybrid_clock.cc:648] HybridClock initialized: now 1790665494797917 us; error 66 us; skew 500 ppm
07:04:54.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.798923 1125 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:54.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:54 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665494, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:04:54.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.805605 1125 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
07:04:54.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.812561 1125 webserver.cc:460] Webserver started at http://127.1.19.62:46103/ using document root <none> and password file <none>
07:04:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.813479 1125 fs_manager.cc:359] Metadata directory not provided
07:04:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.813665 1125 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.814083 1125 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.819218 1125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/instance:
07:04:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81852120f907487c86aa28cec2187ce5"
07:04:54.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:54 on dist-test-slave-tfv6"
07:04:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.820351 1125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal/instance:
07:04:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81852120f907487c86aa28cec2187ce5"
07:04:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:54 on dist-test-slave-tfv6"
07:04:54.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.828855 1125 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
07:04:54.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.836807 1143 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:54.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.838241 1125 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
07:04:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.838748 1125 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:04:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81852120f907487c86aa28cec2187ce5"
07:04:54.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:54 on dist-test-slave-tfv6"
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.839264 1125 fs_report.cc:383] FS layout report
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/data
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.903201 1125 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:54.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.911754 1125 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:54.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:54.912423 1125 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:55.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.011469 1125 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:43129
07:04:55.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.011503 1194 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:43129 every 8 connection(s)
07:04:55.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.015534 1125 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/info.pb
07:04:55.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.023217 1195 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:04:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.025614 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 1125
07:04:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.026193 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal/instance
07:04:55.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.041128 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } has no permanent_uuid. Determining permanent_uuid...
07:04:55.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.61@KRBTEST.COM; defaulting to no policy
07:04:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:04:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.61@KRBTEST.COM" created.
07:04:55.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:55 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665494, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
07:04:55.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:04:55.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.61.keytab.
07:04:55.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.61.keytab.
07:04:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.085270 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:04:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data
07:04:55.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/logs
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/info.pb
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:32793
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
07:04:55.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.61.keytab
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:04:55.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:04:55.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.099931 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } has no permanent_uuid. Determining permanent_uuid...
07:04:55.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.103665 1144 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.109313 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:32793: Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:55.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.158955 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } attempt: 1
07:04:55.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.165820 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:32793: Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:55.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.234256 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } attempt: 2
07:04:55.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.238569 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:32793: Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:55.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.330026 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } attempt: 3
07:04:55.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.334219 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:32793: Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:55.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.477689 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } attempt: 4
07:04:55.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.482748 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:32793: Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:55.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.539844 1201 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.541047 1201 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:55.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.541939 1201 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.577479 1201 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:04:55.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.577718 1201 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:04:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.577975 1201 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.578167 1201 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:04:55.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.578472 1201 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:04:55.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.578799 1201 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:04:55.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.579039 1201 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44975/jwks.json
07:04:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.621169 1201 master_runner.cc:386] Master server non-default flags:
07:04:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:04:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:55.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.61.keytab
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:32793
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/info.pb
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/logs
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:04:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.622777 1201 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:55.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.624163 1201 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.648046 1211 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:55.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:55.782195 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } attempt: 5
07:04:55.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.787637 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:32793: Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:56.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:56.335107 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } attempt: 6
07:04:56.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:56.347381 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:32793: Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:57.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.648036 1208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:57.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:55.648813 1209 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:57.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.071584 1210 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1421 milliseconds
07:04:57.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.071717 1201 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:04:57.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.073218 1201 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:57.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.076438 1201 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:57.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.077870 1201 hybrid_clock.cc:648] HybridClock initialized: now 1790665497077813 us; error 79 us; skew 500 ppm
07:04:57.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.078727 1201 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:57 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665497, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:04:57.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.091493 1201 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
07:04:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.097446 1201 webserver.cc:460] Webserver started at http://127.1.19.61:43483/ using document root <none> and password file <none>
07:04:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.098397 1201 fs_manager.cc:359] Metadata directory not provided
07:04:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.098590 1201 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:57.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.099071 1201 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.103971 1201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/instance:
07:04:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2be5ea84509049529284e0e64bc050f0"
07:04:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:57 on dist-test-slave-tfv6"
07:04:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.105172 1201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal/instance:
07:04:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2be5ea84509049529284e0e64bc050f0"
07:04:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:57 on dist-test-slave-tfv6"
07:04:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.112964 1201 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
07:04:57.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.118805 1224 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.120059 1201 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
07:04:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.120386 1201 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:04:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2be5ea84509049529284e0e64bc050f0"
07:04:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:57 on dist-test-slave-tfv6"
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.120690 1201 fs_report.cc:383] FS layout report
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/data
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:57.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:57.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.186781 1201 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:57.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.193275 1201 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:57.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.193699 1201 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.276047 1201 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:32793
07:04:57.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.276144 1275 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:32793 every 8 connection(s)
07:04:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.278833 1201 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/info.pb
07:04:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.281611 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 1201
07:04:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.282016 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal/instance
07:04:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.287715 1276 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:04:57.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.60@KRBTEST.COM; defaulting to no policy
07:04:57.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:04:57.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.60@KRBTEST.COM" created.
07:04:57.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.301148 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } has no permanent_uuid. Determining permanent_uuid...
07:04:57.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:57 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665497, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
07:04:57.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:04:57.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.60.keytab.
07:04:57.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.60.keytab.
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.331979 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/logs
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/info.pb
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:36411
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.60.keytab
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:04:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:04:57.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:04:57.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.341360 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } has no permanent_uuid. Determining permanent_uuid...
07:04:57.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:57 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665497, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
07:04:57.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.380642 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } has no permanent_uuid. Determining permanent_uuid...
07:04:57.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.383365 1225 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.388659 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.407945 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } attempt: 7
07:04:57.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:57 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665494, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
07:04:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.438218 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 1
07:04:57.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.438715 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } has no permanent_uuid. Determining permanent_uuid...
07:04:57.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.443310 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.445204 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.503645 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 1
07:04:57.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.508229 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.511704 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 2
07:04:57.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.516407 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.589685 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 2
07:04:57.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.594586 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.607854 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 3
07:04:57.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.611474 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.680130 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 3
07:04:57.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.684259 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.709794 1282 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:04:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.710417 1282 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:04:57.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.711028 1282 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:04:57.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.742640 1282 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:04:57.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.742983 1282 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:04:57.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.743295 1282 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:04:57.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.743499 1282 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:04:57.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.743685 1282 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:04:57.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.743877 1282 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:04:57.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.744014 1282 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44975/jwks.json
07:04:57.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.754982 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 4
07:04:57.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.758958 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.782824 1282 master_runner.cc:386] Master server non-default flags:
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.60.keytab
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:36411
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/info.pb
07:04:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/logs
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:04:57.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.783980 1282 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:04:57.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.784994 1282 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:04:57.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.806382 1292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:57.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.824744 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 4
07:04:57.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.828325 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:58.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:58.058405 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 5
07:04:58.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:58.062845 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:58.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:58.111972 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 5
07:04:58.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:58.115986 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:58.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:58.610456 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 6
07:04:58.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:58.620755 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:58.668385 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } attempt: 6
07:04:58.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:58.675460 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36411: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36411: connect: Connection refused (error 111)
07:04:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.807859 1295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:57.808530 1282 server_base.cc:1034] running on GCE node
07:04:59.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:57.806556 1293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:04:59.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.126526 1282 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:04:59.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.129112 1282 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:04:59.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.130539 1282 hybrid_clock.cc:648] HybridClock initialized: now 1790665499130496 us; error 67 us; skew 500 ppm
07:04:59.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.131563 1282 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:04:59.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665499, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:04:59.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.138019 1282 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
07:04:59.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.145082 1282 webserver.cc:460] Webserver started at http://127.1.19.60:45275/ using document root <none> and password file <none>
07:04:59.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.146449 1282 fs_manager.cc:359] Metadata directory not provided
07:04:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.146756 1282 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:04:59.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.147330 1282 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:04:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.154857 1282 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/instance:
07:04:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "635e36511a7a441a9ef4fc65aa0fd290"
07:04:59.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:59 on dist-test-slave-tfv6"
07:04:59.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.156654 1282 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal/instance:
07:04:59.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "635e36511a7a441a9ef4fc65aa0fd290"
07:04:59.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:59 on dist-test-slave-tfv6"
07:04:59.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.168267 1282 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.011s sys 0.001s
07:04:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.179431 1309 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:04:59.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.181110 1282 fs_manager.cc:725] Time spent opening block manager: real 0.008s user 0.003s sys 0.005s
07:04:59.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.181595 1282 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:04:59.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "635e36511a7a441a9ef4fc65aa0fd290"
07:04:59.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:59 on dist-test-slave-tfv6"
07:04:59.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.182073 1282 fs_report.cc:383] FS layout report
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/data
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:04:59.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:04:59.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.227349 1282 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:04:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.234889 1282 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:04:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.235322 1282 kserver.cc:163] Server-wide thread pool size limit: 3276
07:04:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.323304 1282 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:36411
07:04:59.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.323402 1360 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:36411 every 8 connection(s)
07:04:59.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.326248 1282 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/info.pb
07:04:59.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.332746 1361 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:04:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.336040 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 1282
07:04:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.336505 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal/instance
07:04:59.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.351436 1361 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } has no permanent_uuid. Determining permanent_uuid...
07:04:59.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665499, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
07:04:59.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665492, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
07:04:59.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.400731 1361 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } has no permanent_uuid. Determining permanent_uuid...
07:04:59.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665499, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
07:04:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665492, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
07:04:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.430853 1361 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } has no permanent_uuid. Determining permanent_uuid...
07:04:59.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665499, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
07:04:59.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665492, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
07:04:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.464514 1361 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: Bootstrap starting.
07:04:59.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.471933 1361 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: Neither blocks nor log segments found. Creating new log.
07:04:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.474151 1361 log.cc:824] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: Log is configured to *not* fsync() on all Append() calls
07:04:59.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.474208 1100 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:04:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.479674 1361 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: No bootstrap required, opened a new log
07:04:59.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:04:59.483 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:04:59.497 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:04:59.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:04:59.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.62"
port: 43129
}
bound_http_address {
host: "127.1.19.62"
port: 46103
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.61"
port: 32793
}
bound_http_address {
host: "127.1.19.61"
port: 43483
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.60"
port: 36411
}
bound_http_address {
host: "127.1.19.60"
port: 45275
}
}
}
07:04:59.504 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:04:59.505 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:04:59.505 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:04:59.506 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:04:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.513078 1361 raft_consensus.cc:348] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.513826 1361 raft_consensus.cc:374] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:04:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.514158 1361 raft_consensus.cc:729] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 635e36511a7a441a9ef4fc65aa0fd290, State: Initialized, Role: FOLLOWER
07:04:59.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.515115 1361 consensus_queue.cc:260] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:04:59.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.518723 1371 sys_catalog.cc:455] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } } }
07:04:59.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.519974 1371 sys_catalog.cc:458] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [sys.catalog]: This master's current role is: FOLLOWER
07:04:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.520826 1361 sys_catalog.cc:564] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [sys.catalog]: configured and running, proceeding with master startup.
07:04:59.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.528638 1371 raft_consensus.cc:482] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:04:59.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.529265 1371 raft_consensus.cc:504] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:04:59.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.535017 1371 leader_election.cc:288] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129), 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793)
07:04:59.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665499, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:04:59.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:04:59.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:04:59.561 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc
07:04:59.571 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:04:59.573 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc
07:04:59.579 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:04:59.590 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:04:59.592 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:04:59.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:04:59.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:59.593111 1383 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:04:59.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:59.593468 1383 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:04:59.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.564880 1282 master_runner.cc:186] Error getting master registration for 127.1.19.62:43129: OK, instance_id { permanent_uuid: "81852120f907487c86aa28cec2187ce5" instance_seqno: 1790665494985168 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:04:59.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.625854 1282 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:04:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:04:59 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665499, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:04:59.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:04:59.639 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:04:59.640 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc
07:04:59.642 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:04:59.642 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.61:32793
07:04:59.645 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:04:59.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.646206 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 1201
07:04:59.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:59.675021 1310 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.1.19.61:32793 (error 108)
07:04:59.676 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:04:59.676 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.62:43129
07:04:59.677 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:04:59.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.678067 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 1125
07:04:59.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:04:59.683332 1310 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793): Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:04:59.716 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:04:59.716 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.60:36411
07:04:59.717 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:04:59.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.717340 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 1282
07:04:59.739 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:04:59.739 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.61:32793
07:04:59.741 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:04:59.742586 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/logs
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/info.pb
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:04:59.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:32793
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43483
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.61.keytab
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:04:59.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:04:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:04:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:04:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:04:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:04:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:04:59.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:05:00.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.070226 1388 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:00.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.071237 1388 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:00.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.072615 1388 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:00.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.114398 1388 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:05:00.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.114744 1388 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:05:00.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.115263 1388 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:00.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.115610 1388 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:05:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.115938 1388 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:05:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.116252 1388 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:05:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.116482 1388 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44975/jwks.json
07:05:00.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:00.154278 1388 master_runner.cc:386] Master server non-default flags:
07:05:00.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:00.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.61.keytab
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:32793
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/info.pb
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43483
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:00.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/logs
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:05:00.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:00.155891 1388 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:00.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:00.157090 1388 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:05:00.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.177032 1395 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.178354 1398 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:01.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:00.177258 1396 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:01.473717 1397 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:05:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.473837 1388 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:05:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.477927 1388 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:01.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.480651 1388 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:01.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.482095 1388 hybrid_clock.cc:648] HybridClock initialized: now 1790665501482057 us; error 43 us; skew 500 ppm
07:05:01.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.483064 1388 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:01.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:05:01 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665501, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:05:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.489221 1388 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
07:05:01.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.495409 1388 webserver.cc:460] Webserver started at http://127.1.19.61:43483/ using document root <none> and password file <none>
07:05:01.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.496433 1388 fs_manager.cc:359] Metadata directory not provided
07:05:01.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.496690 1388 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:01.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.505276 1388 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
07:05:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.510125 1409 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:01.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.511507 1388 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:05:01.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.511962 1388 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2be5ea84509049529284e0e64bc050f0"
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:57 on dist-test-slave-tfv6"
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.514820 1388 fs_report.cc:383] FS layout report
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/data
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:01.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:01.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:01.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.572191 1388 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:05:01.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.579304 1388 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:01.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.579777 1388 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.658981 1388 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:32793
07:05:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.659091 1461 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:32793 every 8 connection(s)
07:05:01.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.662458 1388 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/data/info.pb
07:05:01.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.666625 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 1388
07:05:01.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:05:01.667 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.62:43129
07:05:01.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:05:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.669413 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:05:01.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/logs
07:05:01.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/info.pb
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:43129
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46103
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.62.keytab
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:05:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:01.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:05:01.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:05:01.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.673785 1462 sys_catalog.cc:263] Verifying existing consensus state
07:05:01.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 07:05:01.674813 1462 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:05:01.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.675325 1388 master.cc:537] Master@127.1.19.61:32793 shutting down...
07:05:01.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:01.736876 1388 master.cc:552] Master@127.1.19.61:32793 shutdown complete.
07:05:01.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:05:02.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.000649 1464 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:02.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.001183 1464 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:02.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.001842 1464 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:02.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.033510 1464 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:05:02.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.033759 1464 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:05:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.034016 1464 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.034207 1464 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:05:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.034425 1464 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:05:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.034607 1464 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:05:02.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.034755 1464 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44975/jwks.json
07:05:02.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:02.073405 1464 master_runner.cc:386] Master server non-default flags:
07:05:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:05:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:02.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.62.keytab
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:43129
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/info.pb
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46103
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/logs
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:02.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:05:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:05:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:05:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:02.074780 1464 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:02.076118 1464 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:05:02.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.098204 1470 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:03.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.099009 1471 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:03.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:02.098244 1473 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:03.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:02.100252 1464 server_base.cc:1034] running on GCE node
07:05:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.353011 1464 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:03.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.359340 1464 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:03.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.360860 1464 hybrid_clock.cc:648] HybridClock initialized: now 1790665503360689 us; error 180 us; skew 500 ppm
07:05:03.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.362023 1464 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:03.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:05:03 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665503, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:05:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.374115 1464 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
07:05:03.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.380023 1464 webserver.cc:460] Webserver started at http://127.1.19.62:46103/ using document root <none> and password file <none>
07:05:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.381304 1464 fs_manager.cc:359] Metadata directory not provided
07:05:03.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.381520 1464 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:03.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.389461 1464 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
07:05:03.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.394608 1482 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:03.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.395927 1464 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:05:03.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.396245 1464 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:05:03.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "81852120f907487c86aa28cec2187ce5"
07:05:03.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:54 on dist-test-slave-tfv6"
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.398756 1464 fs_report.cc:383] FS layout report
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/data
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:03.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.441327 1464 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:05:03.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.448335 1464 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.448920 1464 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.528333 1464 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:43129
07:05:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.528466 1533 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:43129 every 8 connection(s)
07:05:03.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.531810 1464 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/data/info.pb
07:05:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.541497 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 1464
07:05:03.542 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:05:03.543 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.60:36411
07:05:03.543 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:05:03.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.544080 1534 sys_catalog.cc:263] Verifying existing consensus state
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260929 07:05:03.545183 1534 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.545038 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:05:03.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/logs
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/info.pb
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:36411
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45275
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.60.keytab
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:05:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.545727 1464 master.cc:537] Master@127.1.19.62:43129 shutting down...
07:05:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.628741 1464 master.cc:552] Master@127.1.19.62:43129 shutdown complete.
07:05:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:05:03.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.899387 1536 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:05:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.899972 1536 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:05:03.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.900617 1536 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:05:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.934530 1536 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:05:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.934764 1536 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:05:03.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.935019 1536 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:05:03.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.935213 1536 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:05:03.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.935385 1536 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:05:03.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.935577 1536 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:05:03.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.935809 1536 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44975/jwks.json
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.974712 1536 master_runner.cc:386] Master server non-default flags:
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:40865
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:43129,127.1.19.61:32793,127.1.19.60:36411
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/krb5kdc/kudu_127.1.19.60.keytab
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:36411
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44975/jwks.json
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/info.pb
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45275
07:05:03.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:05:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:05:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:05:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:05:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:05:03.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/logs
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 29 Sep 2026 06:22:16 UTC on ccycloud.quasar-vfgaiq.root.comops.site
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4679343
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.975855 1536 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:05:03.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:03.976927 1536 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:05:03.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.994231 1545 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.994282 1542 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:05.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:03.995199 1543 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:05:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:05.262754 1544 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:05:05.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.262840 1536 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:05:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.267639 1536 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:05:05.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.271103 1536 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:05:05.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.272643 1536 hybrid_clock.cc:648] HybridClock initialized: now 1790665505272609 us; error 71 us; skew 500 ppm
07:05:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.273521 1536 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:05:05.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:05:05 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665505, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:05:05.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.279817 1536 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
07:05:05.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.285759 1536 webserver.cc:460] Webserver started at http://127.1.19.60:45275/ using document root <none> and password file <none>
07:05:05.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.286829 1536 fs_manager.cc:359] Metadata directory not provided
07:05:05.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.287056 1536 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:05:05.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.296037 1536 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.000s sys 0.004s
07:05:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.301188 1555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:05:05.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.303325 1536 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.303653 1536 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data,/tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "635e36511a7a441a9ef4fc65aa0fd290"
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-29 07:04:59 on dist-test-slave-tfv6"
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.305943 1536 fs_report.cc:383] FS layout report
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/wal
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/data
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:05:05.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:05:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:05:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:05:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:05:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:05:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:05:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:05:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:05:05.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:05:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.382123 1536 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:05:05.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.388845 1536 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:05:05.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.389276 1536 kserver.cc:163] Server-wide thread pool size limit: 3276
07:05:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.475225 1606 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:36411 every 8 connection(s)
07:05:05.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.475304 1536 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:36411
07:05:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.478494 1536 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskzJbp2h/test-tmp/mini-kudu-cluster3029412007034440918/master-2/data/info.pb
07:05:05.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.488394 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu as pid 1536
07:05:05.489 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:05:05.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.492005 1607 sys_catalog.cc:263] Verifying existing consensus state
07:05:05.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.500299 1607 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: Bootstrap starting.
07:05:05.543 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.550043 1607 log.cc:824] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: Log is configured to *not* fsync() on all Append() calls
07:05:05.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.565572 1607 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:05:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.566510 1607 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: Bootstrap complete.
07:05:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.609579 1607 raft_consensus.cc:348] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.613059 1607 raft_consensus.cc:374] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:05:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.613438 1607 raft_consensus.cc:729] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 635e36511a7a441a9ef4fc65aa0fd290, State: Initialized, Role: FOLLOWER
07:05:05.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.615231 1607 consensus_queue.cc:260] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.622586 1607 sys_catalog.cc:564] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [sys.catalog]: configured and running, proceeding with master startup.
07:05:05.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.620683 1613 sys_catalog.cc:455] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } } }
07:05:05.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.623809 1613 sys_catalog.cc:458] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [sys.catalog]: This master's current role is: FOLLOWER
07:05:05.630 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:05.633 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.60:36411(127.1.19.60:36411)] connecting to peer
07:05:05.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:05.646158 1561 tablet.cc:2369] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:05:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:05.660975 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:05.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:05.661532 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:05.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.691849 1613 raft_consensus.cc:482] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.692391 1613 raft_consensus.cc:504] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:05.696 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:05.697 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:05.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.695679 1613 leader_election.cc:288] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129), 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793)
07:05:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:05.698356 1557 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111)
07:05:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:05.703217 1557 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129): Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111)
07:05:05.705 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:05.706 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:05.707 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:05.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:05.710 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.19.60:36411(127.1.19.60:36411)] Successfully connected to peer
07:05:05.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.660213 1536 master_runner.cc:186] Error getting master registration for 127.1.19.62:43129: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111),
07:05:05.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.717438 1536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:05.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:05.726635 1556 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793): Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:05:05.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.727186 1556 leader_election.cc:302] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 635e36511a7a441a9ef4fc65aa0fd290; no voters: 2be5ea84509049529284e0e64bc050f0, 81852120f907487c86aa28cec2187ce5
07:05:05.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:05.728052 1613 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:05:05.797 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:05:05.805 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.60
07:05:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:05:06 dist-test-slave-tfv6 krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665499, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
07:05:06.048 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xb44afe60, L:/127.0.0.1:36658 - R:/127.1.19.60:36411] using SASL/GSSAPI
07:05:06.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.077 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.085 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.085 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.088 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.088 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.093 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.114 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.117 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.118 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.125 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.128 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.129 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.154 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.155 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.157 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.157 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.158 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.160 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.160 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.161 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.163 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.174 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.175 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.177 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.177 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.177 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.181 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.182 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.184 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.214 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.215 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.217 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.217 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.219 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.219 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.220 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.254 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.256 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.257 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.257 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.258 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.260 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.261 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.263 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.314 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.316 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.317 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.318 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.318 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.320 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.320 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.320 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.325 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.496 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.496 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.497 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.498 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.501 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.501 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.503 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:06.664687 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:06.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:06.665012 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:06.834 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:06.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.837 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:06.837 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:06.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.842 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:06.847 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:07.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:07.439883 1635 raft_consensus.cc:482] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:07.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:07.440335 1635 raft_consensus.cc:504] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:07.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:07.442871 1635 leader_election.cc:288] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129), 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793)
07:05:07.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:07.449288 1557 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129): Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111)
07:05:07.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:07.449797 1556 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793): Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:05:07.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:07.450168 1556 leader_election.cc:302] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 635e36511a7a441a9ef4fc65aa0fd290; no voters: 2be5ea84509049529284e0e64bc050f0, 81852120f907487c86aa28cec2187ce5
07:05:07.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:07.450783 1635 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:05:07.594 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:07.596 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:07.597 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:07.597 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:07.598 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:07.599 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:07.599 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:07.600 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:07.604 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:07.668067 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:07.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:07.668373 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:08.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:08.671702 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:08.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:08.672039 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:08.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:08.979185 1641 raft_consensus.cc:482] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:08.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:08.979595 1641 raft_consensus.cc:504] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:08.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:08.981069 1641 leader_election.cc:288] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129), 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793)
07:05:08.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:08.987464 1557 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129): Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111)
07:05:08.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:08.988132 1556 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793): Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:05:08.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:08.991161 1556 leader_election.cc:302] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 635e36511a7a441a9ef4fc65aa0fd290; no voters: 2be5ea84509049529284e0e64bc050f0, 81852120f907487c86aa28cec2187ce5
07:05:08.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:08.991974 1641 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:05:09.614 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:09.615 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:09.616 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:09.617 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:09.617 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:09.618 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:09.619 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:09.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:09.624 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:09.675587 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:09.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:09.675861 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:10.678645 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:10.678893 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:10.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:10.720280 1536 master_runner.cc:186] Error getting master registration for 127.1.19.62:43129: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111),
07:05:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:10.720727 1536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:10.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:10.853288 1647 raft_consensus.cc:482] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:10.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:10.853703 1647 raft_consensus.cc:504] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:10.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:10.855553 1647 leader_election.cc:288] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129), 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793)
07:05:10.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:10.856719 1557 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:05:10.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:10.859436 1557 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129): Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111)
07:05:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:10.859905 1556 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793): Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:05:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:10.860208 1556 leader_election.cc:302] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 635e36511a7a441a9ef4fc65aa0fd290; no voters: 2be5ea84509049529284e0e64bc050f0, 81852120f907487c86aa28cec2187ce5
07:05:10.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:10.860805 1647 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:05:11.074 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:11.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:11.076 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:11.076 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:11.077 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:11.078 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:11.078 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:11.078 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:11.081 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:11.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:11.681888 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:11.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:11.682150 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:12.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:12.685046 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:12.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:12.685303 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:13.096973 1651 raft_consensus.cc:482] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:13.097378 1651 raft_consensus.cc:504] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:13.098883 1651 leader_election.cc:288] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129), 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793)
07:05:13.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:13.104106 1557 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129): Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111)
07:05:13.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:13.104533 1556 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793): Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:05:13.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:13.104907 1556 leader_election.cc:302] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 635e36511a7a441a9ef4fc65aa0fd290; no voters: 2be5ea84509049529284e0e64bc050f0, 81852120f907487c86aa28cec2187ce5
07:05:13.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:13.105612 1651 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:05:13.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:13.688133 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:13.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:13.688396 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:14.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:14.375 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:14.376 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:14.377 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:14.377 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:14.378 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:14.378 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:14.378 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:14.381 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:14.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:14.692171 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:14.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:14.692538 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:15.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:05:15 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665515, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:05:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:15.283727 1551 init.cc:295] Successfully reacquired a new kerberos TGT
07:05:15.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:15.695462 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:15.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:15.695721 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:15.724860 1536 master_runner.cc:186] Error getting master registration for 127.1.19.62:43129: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111),
07:05:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:15.725335 1536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:16.694 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:16.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:16.697 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:16.698 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:16.698 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:16.699 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:16.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:16.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:16.699635 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:16.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:16.699940 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:16.702 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:17.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:17.703182 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:17.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:17.703472 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:18.334 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:18.335 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:18.336 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:18.337 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:18.337 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:18.337 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:18.337 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:18.338 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:18.341 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:18.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:18.706280 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:18.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:18.706657 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:19.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:19.709617 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:19.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:19.709949 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:20.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:20.713228 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:20.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:20.713569 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:20.728237 1536 master_runner.cc:186] Error getting master registration for 127.1.19.62:43129: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111),
07:05:20.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:20.728744 1536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:21.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:21.036267 1665 raft_consensus.cc:482] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:21.036824 1665 raft_consensus.cc:504] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:21.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:21.038589 1665 leader_election.cc:288] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129), 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793)
07:05:21.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:21.039968 1557 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111) [suppressed 3 similar messages]
07:05:21.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:21.043367 1557 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129): Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111)
07:05:21.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:21.044075 1556 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793): Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:05:21.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:21.044384 1556 leader_election.cc:302] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 635e36511a7a441a9ef4fc65aa0fd290; no voters: 2be5ea84509049529284e0e64bc050f0, 81852120f907487c86aa28cec2187ce5
07:05:21.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:21.044986 1665 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:05:21.694 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:21.695 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:21.696 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:21.696 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:21.696 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:21.697 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:21.697 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:21.697 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:21.700 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:21.716822 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:21.717164 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:22.720062 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:22.720364 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:23.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:23.324069 605 debug-util.cc:398] Leaking SignalData structure 0x7b0800081ae0 after lost signal to thread 536
07:05:23.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:23.324843 605 debug-util.cc:398] Leaking SignalData structure 0x7b0800081c40 after lost signal to thread 608
07:05:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:23.723213 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:23.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:23.723577 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:24.727784 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:24.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:24.728207 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:25.094 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:25.095 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:25.096 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:25.097 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:25.097 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:25.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:25.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:25.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:25.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:25.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 29 07:05:25 dist-test-slave-tfv6 krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790665525, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:05:25.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:25.286164 1551 init.cc:295] Successfully reacquired a new kerberos TGT
07:05:25.293 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:25.295 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:25.295 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:25.295 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:25.296 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:25.297 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:25.298 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:25.298 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:25.300 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:25.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:25.679291 1672 raft_consensus.cc:482] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:05:25.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:25.679737 1672 raft_consensus.cc:504] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "81852120f907487c86aa28cec2187ce5" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 43129 } } peers { permanent_uuid: "2be5ea84509049529284e0e64bc050f0" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 32793 } } peers { permanent_uuid: "635e36511a7a441a9ef4fc65aa0fd290" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36411 } }
07:05:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:25.681664 1672 leader_election.cc:288] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129), 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793)
07:05:25.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:25.687397 1557 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 81852120f907487c86aa28cec2187ce5 (127.1.19.62:43129): Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111)
07:05:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:25.687951 1556 leader_election.cc:334] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2be5ea84509049529284e0e64bc050f0 (127.1.19.61:32793): Network error: Client connection negotiation failed: client connection to 127.1.19.61:32793: connect: Connection refused (error 111)
07:05:25.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:25.688385 1556 leader_election.cc:302] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 635e36511a7a441a9ef4fc65aa0fd290; no voters: 2be5ea84509049529284e0e64bc050f0, 81852120f907487c86aa28cec2187ce5
07:05:25.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:25.689249 1672 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:05:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:25.730932 1536 master_runner.cc:186] Error getting master registration for 127.1.19.62:43129: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111),
07:05:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:25.731508 1536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:25.731913 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:25.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:25.732124 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:26.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:26.735313 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:26.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:26.735638 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:27.738427 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:27.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:27.738682 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:28.654 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:28.657 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:28.657 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:28.658 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:28.658 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:28.659 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:28.659 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:28.659 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:28.662 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:28.741850 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:28.742204 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:29.034 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:29.035 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:29.035 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:29.035 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:29.036 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:29.036 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:29.037 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:29.037 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:29.039 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:29.533 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:29.535 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:29.535 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:29.535 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:29.536 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:29.536 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:29.536 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:29.537 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:29.539 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:29.745611 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:29.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:29.745914 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:30.633 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:30.634 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:30.635 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:30.635 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:30.635 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:30.635 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:30.635 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:30.636 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:30.638 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:30.734135 1536 master_runner.cc:186] Error getting master registration for 127.1.19.62:43129: Network error: Client connection negotiation failed: client connection to 127.1.19.62:43129: connect: Connection refused (error 111),
07:05:30.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:30.734550 1536 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:05:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:30.748749 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:30.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:30.749038 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:31.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:31.751839 1626 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 635e36511a7a441a9ef4fc65aa0fd290: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:05:31.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:31.752100 1626 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:05:32.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:32.225608 1603 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 1537
07:05:32.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:32.226118 1603 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb620 after lost signal to thread 1606
07:05:32.454 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:32793(127.1.19.61:32793)] connecting to peer
07:05:32.455 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:43129(127.1.19.62:43129)] connecting to peer
07:05:32.456 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:32.456 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:32793(127.1.19.61:32793)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:32.456 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:32793: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793
07:05:32.457 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:32.457 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:43129(127.1.19.62:43129)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:32.457 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:43129: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
07:05:32.460 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411; will retry
07:05:32.467 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26968), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [202ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [210ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [574ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [578ms] refreshing cache from master, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [589ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [592ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [630ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [633ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [668ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [766ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [822ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [824ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [828ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1002ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1008ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1338ms] refreshing cache from master, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1350ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1351ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2098ms] refreshing cache from master, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [2102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [2103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [2103ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2108ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [2108ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4117ms] refreshing cache from master, [4117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [4118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [4121ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [4123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [4124ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4128ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [4128ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [5577ms] refreshing cache from master, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [5580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [5581ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
07:05:32.472 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.60:36411(127.1.19.60:36411)] disconnecting while in state READY
07:05:32.474 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.62:43129(127.1.19.62:43129)] disconnecting while in state TERMINATED
07:05:32.474 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.61:32793(127.1.19.61:32793)] disconnecting while in state TERMINATED
07:05:32.479 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:05:32.481 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:05:32.492 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.19.60:36411(127.1.19.60:36411)] handling channelInactive
07:05:32.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.60:36411(127.1.19.60:36411)] cleaning up while in state READY due to: connection closed
07:05:32.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:32.487530 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 1536
07:05:33.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T07:05:33Z chronyd exiting
07:05:34.720 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:05:34.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:34.721925 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 653
07:05:34.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:34.748289 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 812
07:05:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:34.775130 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 949
07:05:34.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:34.802891 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 386
07:05:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:34.834911 559 connection.cc:535] client connection to 127.0.93.190:41871 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:34.835783 559 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:05:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:34.835897 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 456
07:05:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260929 07:05:34.842366 559 consensus_peers.cc:487] T 00000000000000000000000000000000 P ff5bf1afc1c6482293e95c0dbba87089 -> Peer 838638ab6fe84c2c9e6d76785afe833f (127.0.93.190:41871): Couldn't send request to peer 838638ab6fe84c2c9e6d76785afe833f. Status: Network error: Client connection negotiation failed: client connection to 127.0.93.190:41871: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
07:05:34.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260929 07:05:34.871253 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskzJbp2h/build/tsan/bin/kudu with pid 531
07:05:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-29T07:05:36Z chronyd exiting
07:05:37.054 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26968), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [202ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [210ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [574ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [578ms] refreshing cache from master, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [589ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [592ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [630ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [633ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [668ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [766ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [822ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [824ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [828ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1002ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1008ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1338ms] refreshing cache from master, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1350ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1351ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2098ms] refreshing cache from master, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [2102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [2103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [2103ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2108ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [2108ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4117ms] refreshing cache from master, [4117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [4118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [4121ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [4123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [4124ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4128ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [4128ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [5577ms] refreshing cache from master, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [5580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [5581ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.62:43129
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26968), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [202ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [210ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [574ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [578ms] refreshing cache from master, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [589ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [592ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [630ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [633ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [668ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [766ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [822ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [824ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [828ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1002ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1008ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1338ms] refreshing cache from master, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1350ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1351ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2098ms] refreshing cache from master, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [2102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [2103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [2103ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2108ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [2108ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4117ms] refreshing cache from master, [4117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [4118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [4121ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [4123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [4124ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4128ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [4128ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [5577ms] refreshing cache from master, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [5580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [5581ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.62:43129
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
07:05:37.580 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
07:05:37.580 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26968), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [202ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [210ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [574ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [578ms] refreshing cache from master, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [589ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [592ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [630ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [633ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [668ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [766ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [822ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [824ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [828ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1002ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1008ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1338ms] refreshing cache from master, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1350ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1351ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2098ms] refreshing cache from master, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [2102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [2103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [2103ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2108ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [2108ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4117ms] refreshing cache from master, [4117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [4118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [4121ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [4123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [4124ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4128ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [4128ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [5577ms] refreshing cache from master, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [5580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [5581ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.62:43129
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=26968), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [202ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [210ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [569ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [574ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [578ms] refreshing cache from master, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [589ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [592ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [596ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [598ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [617ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [628ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [630ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [633ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [657ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [661ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [664ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [666ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [668ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [685ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [687ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [688ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [717ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [721ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [723ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [726ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [727ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [757ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [761ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [766ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [817ms] refreshing cache from master, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [820ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [822ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [823ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [824ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [828ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [829ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [997ms] refreshing cache from master, [997ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1000ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1000ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1002ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1005ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1007ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1008ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1338ms] refreshing cache from master, [1338ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [1341ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [1344ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [1345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [1350ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [1351ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2098ms] refreshing cache from master, [2098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [2101ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [2102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [2103ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [2103ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [2108ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [2108ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4117ms] refreshing cache from master, [4117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [4118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [4121ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:32793: Network error: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793, [4123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [4124ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:43129: Network error: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [4128ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36411: OK, [4128ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129, [5577ms] refreshing cache from master, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:32793, [5578ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:43129, [5580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36411, [5581ms] too many traces: truncated at 100 traces, deferred=Deferred@946387636(state=PENDING, result=null, callback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879), errback=(continuation of Deferred@995851181 after retry RPC after error@1323589291) -> (continuation of Deferred@739257152 after retry RPC after error@1372625260) -> (continuation of Deferred@825628392 after retry RPC after error@1573590883) -> (continuation of Deferred@1890352561 after retry RPC after error@635605297) -> (continuation of Deferred@610419251 after retry RPC after error@555170259) -> (continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:32793,127.1.19.62:43129,127.1.19.60:36411) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:32793(127.1.19.61:32793): Connection refused: /127.1.19.61:32793,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:43129(127.1.19.62:43129): Connection refused: /127.1.19.62:43129
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.62:43129
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)