07:46:19.556 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:46:19.568 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task9m7ByE/test-tmp
07:46:19.571 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:46:19.575 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task9m7ByE/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:46:19.826 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:46:19.973 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:46:19.974 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:46:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:19.977632 22646 mini_oidc.cc:152] Starting JWKS server
07:46:19.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:19.981529 22646 webserver.cc:460] Webserver started at http://127.0.0.1:34415/ using document root <none> and password file <none>
07:46:19.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:19.982017 22646 mini_oidc.cc:189] Starting OIDC Discovery server
07:46:19.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:19.984457 22646 webserver.cc:460] Webserver started at http://127.0.0.1:36503/ using document root <none> and password file <none>
07:46:20.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:46:20Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:46:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:46:20Z Disabled control of system clock
07:46:20.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:20.040072 22646 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:20.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:20.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/wal
07:46:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/data
07:46:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:20.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/logs
07:46:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/data/info.pb
07:46:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:20.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:20.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.190:34791
07:46:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.190
07:46:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34415/jwks.json with env {}
07:46:20.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.353132 22658 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.353684 22658 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:20.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.354280 22658 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:20.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.391093 22658 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.391376 22658 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.391592 22658 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.391781 22658 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.392038 22658 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:20.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.392236 22658 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34415/jwks.json
07:46:20.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:20.425938 22658 master_runner.cc:386] Master server non-default flags:
07:46:20.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:20.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:20.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:20.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:20.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/data
07:46:20.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/wal
07:46:20.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:20.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:20.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:20.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:20.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.190:34791
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34415/jwks.json
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/data/info.pb
07:46:20.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.190
07:46:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/logs
07:46:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:20.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:20.427271 22658 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:20.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:20.428879 22658 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:20.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.447907 22664 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:21.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.449067 22667 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:21.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:20.449909 22658 server_base.cc:1034] running on GCE node
07:46:21.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:20.449353 22665 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:21.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.644495 22658 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:21.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.647535 22658 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:21.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.648944 22658 hybrid_clock.cc:648] HybridClock initialized: now 1790235981648890 us; error 68 us; skew 500 ppm
07:46:21.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.649739 22658 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:21.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.662269 22658 webserver.cc:460] Webserver started at http://127.22.29.190:41529/ using document root <none> and password file <none>
07:46:21.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.663120 22658 fs_manager.cc:359] Metadata directory not provided
07:46:21.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.663292 22658 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:21.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.663718 22658 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.668668 22658 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/data/instance:
07:46:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b2ab30c18184068b1c7c986c16daf9c"
07:46:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:21 on dist-test-slave-01fg"
07:46:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.669809 22658 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/wal/instance:
07:46:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b2ab30c18184068b1c7c986c16daf9c"
07:46:21.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:21 on dist-test-slave-01fg"
07:46:21.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.676919 22658 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.000s sys 0.008s
07:46:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.682400 22674 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.683388 22658 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
07:46:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.683686 22658 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/wal
07:46:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b2ab30c18184068b1c7c986c16daf9c"
07:46:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:21 on dist-test-slave-01fg"
07:46:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.683945 22658 fs_report.cc:383] FS layout report
07:46:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/wal
07:46:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/wal
07:46:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/data/data
07:46:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:21.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.716903 22658 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.718211 22658 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:21.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.718575 22658 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:21.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.788071 22658 rpc_server.cc:307] RPC server started. Bound to: 127.22.29.190:34791
07:46:21.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.788163 22725 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.29.190:34791 every 8 connection(s)
07:46:21.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.790333 22658 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/data/info.pb
07:46:21.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.794580 22646 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 22658
07:46:21.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.795075 22646 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-0/wal/instance
07:46:21.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.796484 22726 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.800043 22646 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/wal
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/data
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:21.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/logs
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/data/info.pb
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:21.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.189:42083
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.189
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:21.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:21.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:21.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34415/jwks.json with env {}
07:46:21.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.810439 22726 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } has no permanent_uuid. Determining permanent_uuid...
07:46:21.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.840555 22726 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } has no permanent_uuid. Determining permanent_uuid...
07:46:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:21.842667 22677 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.29.189:42083: connect: Connection refused (error 111)
07:46:21.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:21.846972 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.189:42083: Network error: Client connection negotiation failed: client connection to 127.22.29.189:42083: connect: Connection refused (error 111)
07:46:21.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.896463 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } attempt: 1
07:46:21.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:21.901018 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.189:42083: Network error: Client connection negotiation failed: client connection to 127.22.29.189:42083: connect: Connection refused (error 111)
07:46:21.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:21.969501 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } attempt: 2
07:46:21.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:21.973559 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.189:42083: Network error: Client connection negotiation failed: client connection to 127.22.29.189:42083: connect: Connection refused (error 111)
07:46:22.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:22.064970 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } attempt: 3
07:46:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.068596 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.189:42083: Network error: Client connection negotiation failed: client connection to 127.22.29.189:42083: connect: Connection refused (error 111)
07:46:22.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.106096 22728 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:22.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.106624 22728 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:22.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.107313 22728 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:22.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.137200 22728 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.137477 22728 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.137660 22728 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.137836 22728 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.138103 22728 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:22.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.138334 22728 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34415/jwks.json
07:46:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:22.173527 22728 master_runner.cc:386] Master server non-default flags:
07:46:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/data
07:46:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/wal
07:46:22.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.189:42083
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34415/jwks.json
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:22.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/data/info.pb
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.189
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/logs
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:22.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:22.174800 22728 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:22.176920 22728 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:22.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.198203 22740 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:22.211956 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } attempt: 4
07:46:22.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.216203 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.189:42083: Network error: Client connection negotiation failed: client connection to 127.22.29.189:42083: connect: Connection refused (error 111)
07:46:22.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:22.515645 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } attempt: 5
07:46:22.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.518989 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.189:42083: Network error: Client connection negotiation failed: client connection to 127.22.29.189:42083: connect: Connection refused (error 111)
07:46:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.066602 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } attempt: 6
07:46:23.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.074118 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.189:42083: Network error: Client connection negotiation failed: client connection to 127.22.29.189:42083: connect: Connection refused (error 111)
07:46:23.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.198385 22738 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:23.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:22.200343 22728 server_base.cc:1034] running on GCE node
07:46:23.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:22.198390 22737 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:23.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.397768 22728 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.400413 22728 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:23.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.401774 22728 hybrid_clock.cc:648] HybridClock initialized: now 1790235983401737 us; error 51 us; skew 500 ppm
07:46:23.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.402864 22728 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:23.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.409899 22728 webserver.cc:460] Webserver started at http://127.22.29.189:38711/ using document root <none> and password file <none>
07:46:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.410883 22728 fs_manager.cc:359] Metadata directory not provided
07:46:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.411140 22728 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:23.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.411679 22728 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.416399 22728 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/data/instance:
07:46:23.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "167b3e21b69d49ec883a88729664d949"
07:46:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:23 on dist-test-slave-01fg"
07:46:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.417600 22728 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/wal/instance:
07:46:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "167b3e21b69d49ec883a88729664d949"
07:46:23.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:23 on dist-test-slave-01fg"
07:46:23.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.425400 22728 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.003s
07:46:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.430661 22749 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:23.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.431659 22728 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
07:46:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.432009 22728 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/wal
07:46:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "167b3e21b69d49ec883a88729664d949"
07:46:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:23 on dist-test-slave-01fg"
07:46:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.432375 22728 fs_report.cc:383] FS layout report
07:46:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/wal
07:46:23.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/wal
07:46:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/data/data
07:46:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:23.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.470402 22728 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:23.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.471925 22728 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:23.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.472441 22728 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.544216 22728 rpc_server.cc:307] RPC server started. Bound to: 127.22.29.189:42083
07:46:23.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.544303 22800 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.29.189:42083 every 8 connection(s)
07:46:23.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.546836 22728 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/data/info.pb
07:46:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.552023 22646 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 22728
07:46:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.552259 22801 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.552446 22646 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-1/wal/instance
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.554662 22646 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/wal
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/data
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/logs
07:46:23.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/data/info.pb
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.188:36219
07:46:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.188
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34415/jwks.json with env {}
07:46:23.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.563864 22801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } has no permanent_uuid. Determining permanent_uuid...
07:46:23.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.587532 22801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } has no permanent_uuid. Determining permanent_uuid...
07:46:23.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.604089 22801 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } has no permanent_uuid. Determining permanent_uuid...
07:46:23.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.606580 22751 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:23.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.610373 22801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:23.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.659785 22801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 1
07:46:23.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.663489 22801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.731841 22801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 2
07:46:23.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.735823 22801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:23.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.827194 22801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 3
07:46:23.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.831444 22801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:23.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.861879 22803 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:23.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.862604 22803 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:23.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.863431 22803 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.893553 22803 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.893808 22803 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.893985 22803 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.894164 22803 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.894423 22803 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:23.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.894562 22803 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34415/jwks.json
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.927910 22803 master_runner.cc:386] Master server non-default flags:
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/data
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/wal
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.188:36219
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34415/jwks.json
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/data/info.pb
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.188
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:23.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/logs
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:23.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.929242 22803 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:23.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.931244 22803 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:23.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.974810 22801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 4
07:46:23.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.978910 22801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:24.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.134527 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } attempt: 7
07:46:24.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.159325 22726 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } has no permanent_uuid. Determining permanent_uuid...
07:46:24.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:24.169438 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:24.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.227841 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 1
07:46:24.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:24.232277 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:24.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.278298 22801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 5
07:46:24.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:24.281744 22801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:24.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.313712 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 2
07:46:24.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:24.318645 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:24.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.404047 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 3
07:46:24.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:24.408421 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.548806 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 4
07:46:24.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:24.554666 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:24.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.829335 22801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 6
07:46:24.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:24.837603 22801 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:24.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:24.838100 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 5
07:46:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:24.842547 22726 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.29.188:36219: Network error: Client connection negotiation failed: client connection to 127.22.29.188:36219: connect: Connection refused (error 111)
07:46:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:23.951597 22803 server_base.cc:1034] running on GCE node
07:46:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.950454 22816 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.950413 22813 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:23.950407 22814 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:25.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.148381 22803 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:25.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.155809 22803 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:25.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.157205 22803 hybrid_clock.cc:648] HybridClock initialized: now 1790235985157174 us; error 42 us; skew 500 ppm
07:46:25.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.158061 22803 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:25.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.164614 22803 webserver.cc:460] Webserver started at http://127.22.29.188:42199/ using document root <none> and password file <none>
07:46:25.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.165537 22803 fs_manager.cc:359] Metadata directory not provided
07:46:25.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.165762 22803 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:25.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.166314 22803 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:25.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.171144 22803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/data/instance:
07:46:25.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a18ab838d0b46768b924746a621589d"
07:46:25.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:25 on dist-test-slave-01fg"
07:46:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.172386 22803 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/wal/instance:
07:46:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a18ab838d0b46768b924746a621589d"
07:46:25.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:25 on dist-test-slave-01fg"
07:46:25.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.179862 22803 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
07:46:25.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.185539 22828 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:25.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.186480 22803 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
07:46:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.186825 22803 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/wal
07:46:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a18ab838d0b46768b924746a621589d"
07:46:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:25 on dist-test-slave-01fg"
07:46:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.187175 22803 fs_report.cc:383] FS layout report
07:46:25.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/wal
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/wal
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/data/data
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:25.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:25.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.217885 22803 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:25.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.219337 22803 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:25.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.219853 22803 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.293236 22803 rpc_server.cc:307] RPC server started. Bound to: 127.22.29.188:36219
07:46:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.293325 22879 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.29.188:36219 every 8 connection(s)
07:46:25.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.295940 22803 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/data/info.pb
07:46:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.302081 22880 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:25.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.304418 22646 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 22803
07:46:25.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.304883 22646 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/master-2/wal/instance
07:46:25.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.322078 22880 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } has no permanent_uuid. Determining permanent_uuid...
07:46:25.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.347846 22880 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } has no permanent_uuid. Determining permanent_uuid...
07:46:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.361351 22880 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } has no permanent_uuid. Determining permanent_uuid...
07:46:25.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.377918 22880 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: Bootstrap starting.
07:46:25.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.384788 22880 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: Neither blocks nor log segments found. Creating new log.
07:46:25.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.386243 22880 log.cc:824] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: Log is configured to *not* fsync() on all Append() calls
07:46:25.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.390923 22880 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: No bootstrap required, opened a new log
07:46:25.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.394940 22726 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 6
07:46:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.417708 22726 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c: Bootstrap starting.
07:46:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.417953 22880 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:25.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.418560 22880 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:46:25.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.418866 22880 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3a18ab838d0b46768b924746a621589d, State: Initialized, Role: FOLLOWER
07:46:25.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.419751 22880 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.422585 22890 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:25.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.423448 22890 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [sys.catalog]: This master's current role is: FOLLOWER
07:46:25.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.423827 22880 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [sys.catalog]: configured and running, proceeding with master startup.
07:46:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.425150 22726 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c: Neither blocks nor log segments found. Creating new log.
07:46:25.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.427439 22726 log.cc:824] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c: Log is configured to *not* fsync() on all Append() calls
07:46:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.435905 22726 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c: No bootstrap required, opened a new log
07:46:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:25.464123 22902 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:25.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:25.464444 22902 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:25.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.468261 22726 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:25.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.468820 22726 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:46:25.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.469107 22726 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b2ab30c18184068b1c7c986c16daf9c, State: Initialized, Role: FOLLOWER
07:46:25.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.469882 22726 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.473245 22904 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:25.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:25.474776 22834 tablet.cc:2369] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:46:25.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.475126 22904 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: This master's current role is: FOLLOWER
07:46:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:25.477253 22680 tablet.cc:2369] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:46:25.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.477438 22726 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: configured and running, proceeding with master startup.
07:46:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.454947 22803 master_runner.cc:186] Error getting master registration for 127.22.29.190:34791: OK, instance_id { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" instance_seqno: 1790235981769248 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:46:25.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.493319 22803 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:25.509645 22915 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:25.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:25.510244 22915 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:25.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.508520 22658 master_runner.cc:186] Error getting master registration for 127.22.29.189:42083: OK, instance_id { permanent_uuid: "167b3e21b69d49ec883a88729664d949" instance_seqno: 1790235983524619 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:46:25.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.532630 22658 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.740187 22904 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.740695 22904 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.743054 22904 leader_election.cc:288] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 167b3e21b69d49ec883a88729664d949 (127.22.29.189:42083), 3a18ab838d0b46768b924746a621589d (127.22.29.188:36219)
07:46:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.744688 22855 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b2ab30c18184068b1c7c986c16daf9c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3a18ab838d0b46768b924746a621589d" is_pre_election: true
07:46:25.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.745452 22855 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b2ab30c18184068b1c7c986c16daf9c in term 0.
07:46:25.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.747568 22676 leader_election.cc:302] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3a18ab838d0b46768b924746a621589d, 8b2ab30c18184068b1c7c986c16daf9c; no voters:
07:46:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.748575 22904 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 0 FOLLOWER]: Leader pre-election won for term 1
07:46:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.748890 22904 raft_consensus.cc:482] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:46:25.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.749161 22904 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 0 FOLLOWER]: Advancing to term 1
07:46:25.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.756044 22904 raft_consensus.cc:504] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:25.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.758179 22904 leader_election.cc:288] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [CANDIDATE]: Term 1 election: Requested vote from peers 167b3e21b69d49ec883a88729664d949 (127.22.29.189:42083), 3a18ab838d0b46768b924746a621589d (127.22.29.188:36219)
07:46:25.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.759984 22855 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b2ab30c18184068b1c7c986c16daf9c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3a18ab838d0b46768b924746a621589d"
07:46:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.760495 22855 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [term 0 FOLLOWER]: Advancing to term 1
07:46:25.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.767046 22855 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8b2ab30c18184068b1c7c986c16daf9c in term 1.
07:46:25.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.768745 22676 leader_election.cc:302] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3a18ab838d0b46768b924746a621589d, 8b2ab30c18184068b1c7c986c16daf9c; no voters:
07:46:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.769611 22904 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 1 FOLLOWER]: Leader election won for term 1
07:46:25.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.781524 22904 raft_consensus.cc:686] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [term 1 LEADER]: Becoming Leader. State: Replica: 8b2ab30c18184068b1c7c986c16daf9c, State: Running, Role: LEADER
07:46:25.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.782810 22904 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.821223 22917 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b2ab30c18184068b1c7c986c16daf9c. Latest consensus state: current_term: 1 leader_uuid: "8b2ab30c18184068b1c7c986c16daf9c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.822067 22917 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: This master's current role is: LEADER
07:46:25.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.831094 22919 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:46:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.839037 22919 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:46:25.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.863477 22855 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [term 1 FOLLOWER]: Refusing update from remote peer 8b2ab30c18184068b1c7c986c16daf9c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:46:25.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.866827 22904 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:46:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.898164 22801 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } attempt: 7
07:46:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.924211 22890 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b2ab30c18184068b1c7c986c16daf9c. Latest consensus state: current_term: 1 leader_uuid: "8b2ab30c18184068b1c7c986c16daf9c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.924964 22890 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [sys.catalog]: This master's current role is: FOLLOWER
07:46:25.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.933372 22920 mvcc.cc:205] Tried to move back new op lower bound from 7332806598075699200 to 7332806597776195584. Current Snapshot: MvccSnapshot[applied={T|T < 7332806598075699200}]
07:46:25.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.937750 22921 mvcc.cc:205] Tried to move back new op lower bound from 7332806598075699200 to 7332806597776195584. Current Snapshot: MvccSnapshot[applied={T|T < 7332806598075699200}]
07:46:25.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.945456 22890 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8b2ab30c18184068b1c7c986c16daf9c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:25.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.946261 22890 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d [sys.catalog]: This master's current role is: FOLLOWER
07:46:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.963557 22904 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8b2ab30c18184068b1c7c986c16daf9c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:25.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.964327 22904 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: This master's current role is: LEADER
07:46:25.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.972994 22919 catalog_manager.cc:1347] Generated new cluster ID: 59ef86b0e1ed41a8a22ec2a9dc61c49e
07:46:25.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.973280 22919 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:46:25.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:25.990483 22801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949: Bootstrap starting.
07:46:26.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.008107 22801 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949: Neither blocks nor log segments found. Creating new log.
07:46:26.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.013834 22801 log.cc:824] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949: Log is configured to *not* fsync() on all Append() calls
07:46:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.021306 22919 catalog_manager.cc:1370] Generated new certificate authority record
07:46:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.023831 22919 catalog_manager.cc:1504] Loading token signing keys...
07:46:26.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.044827 22801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949: No bootstrap required, opened a new log
07:46:26.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.057857 22919 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c: Generated new TSK 0
07:46:26.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.059376 22919 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:46:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.077217 22646 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/wal
07:46:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/data
07:46:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:26.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/logs
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/data/info.pb
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.129:0
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.29.129
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.129
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:26.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:46:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.139014 22801 raft_consensus.cc:348] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:26.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.141079 22801 raft_consensus.cc:374] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:46:26.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.141793 22801 raft_consensus.cc:729] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 167b3e21b69d49ec883a88729664d949, State: Initialized, Role: FOLLOWER
07:46:26.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.143934 22801 consensus_queue.cc:260] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } }
07:46:26.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.160820 22928 sys_catalog.cc:455] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:26.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.161670 22928 sys_catalog.cc:458] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [sys.catalog]: This master's current role is: FOLLOWER
07:46:26.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.166219 22776 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b2ab30c18184068b1c7c986c16daf9c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "167b3e21b69d49ec883a88729664d949" is_pre_election: true
07:46:26.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.167042 22776 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8b2ab30c18184068b1c7c986c16daf9c in term 0.
07:46:26.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.168262 22774 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [term 0 FOLLOWER]: Advancing to term 1
07:46:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.175099 22774 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [term 1 FOLLOWER]: Refusing update from remote peer 8b2ab30c18184068b1c7c986c16daf9c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:46:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.175628 22775 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8b2ab30c18184068b1c7c986c16daf9c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "167b3e21b69d49ec883a88729664d949"
07:46:26.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.177768 22801 sys_catalog.cc:564] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [sys.catalog]: configured and running, proceeding with master startup.
07:46:26.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.178279 22904 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [LEADER]: Connected to new peer: Peer: permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:46:26.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.268867 22928 sys_catalog.cc:455] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8b2ab30c18184068b1c7c986c16daf9c. Latest consensus state: current_term: 1 leader_uuid: "8b2ab30c18184068b1c7c986c16daf9c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:26.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.268739 22936 sys_catalog.cc:455] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8b2ab30c18184068b1c7c986c16daf9c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:26.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.269587 22928 sys_catalog.cc:458] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [sys.catalog]: This master's current role is: FOLLOWER
07:46:26.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.269622 22936 sys_catalog.cc:458] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949 [sys.catalog]: This master's current role is: FOLLOWER
07:46:26.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.303702 22944 catalog_manager.cc:1259] Loaded cluster ID: 59ef86b0e1ed41a8a22ec2a9dc61c49e
07:46:26.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.304874 22944 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949: loading cluster ID for follower catalog manager: success
07:46:26.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.336067 22944 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949: acquiring CA information for follower catalog manager: success
07:46:26.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.342842 22944 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 167b3e21b69d49ec883a88729664d949: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:46:26.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.470285 22902 catalog_manager.cc:1259] Loaded cluster ID: 59ef86b0e1ed41a8a22ec2a9dc61c49e
07:46:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.470897 22902 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: loading cluster ID for follower catalog manager: success
07:46:26.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.490206 22902 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: acquiring CA information for follower catalog manager: success
07:46:26.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.500607 22902 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 3a18ab838d0b46768b924746a621589d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:46:26.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:26.713196 22927 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:26.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:26.713924 22927 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:26.714947 22927 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:26.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.728013 22904 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8b2ab30c18184068b1c7c986c16daf9c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8b2ab30c18184068b1c7c986c16daf9c" member_type: VOTER last_known_addr { host: "127.22.29.190" port: 34791 } } peers { permanent_uuid: "167b3e21b69d49ec883a88729664d949" member_type: VOTER last_known_addr { host: "127.22.29.189" port: 42083 } } peers { permanent_uuid: "3a18ab838d0b46768b924746a621589d" member_type: VOTER last_known_addr { host: "127.22.29.188" port: 36219 } } }
07:46:26.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.729547 22904 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b2ab30c18184068b1c7c986c16daf9c [sys.catalog]: This master's current role is: LEADER
07:46:26.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:26.773118 22927 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:26.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:26.774497 22927 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.29.129
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.885632 22927 tablet_server_runner.cc:78] Tablet server non-default flags:
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/data
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/wal
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.129:0
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/data/info.pb
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.129
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.29.129
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/logs
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:26.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.887212 22927 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:26.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:26.904381 22927 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:26.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:26.951746 22954 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:26.954619 22953 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:28.455360 22955 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.455436 22927 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:46:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:26.964409 22957 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:28.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.460505 22927 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.463531 22927 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:28.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.465065 22927 hybrid_clock.cc:648] HybridClock initialized: now 1790235988465038 us; error 52 us; skew 500 ppm
07:46:28.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.466075 22927 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:28.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.477372 22927 webserver.cc:460] Webserver started at http://127.22.29.129:43561/ using document root <none> and password file <none>
07:46:28.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.479553 22927 fs_manager.cc:359] Metadata directory not provided
07:46:28.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.479816 22927 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:28.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.480728 22927 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.487548 22927 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/data/instance:
07:46:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f1b312598fe488198a0e4305edb5c01"
07:46:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:28 on dist-test-slave-01fg"
07:46:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.488904 22927 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/wal/instance:
07:46:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f1b312598fe488198a0e4305edb5c01"
07:46:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:28 on dist-test-slave-01fg"
07:46:28.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.497780 22927 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
07:46:28.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.505153 22963 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:28.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.506438 22927 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.506785 22927 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/wal
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f1b312598fe488198a0e4305edb5c01"
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:28 on dist-test-slave-01fg"
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.507162 22927 fs_report.cc:383] FS layout report
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/wal
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/wal
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/data/data
07:46:28.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:28.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:28.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.565393 22927 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:28.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.567334 22927 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:28.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.567917 22927 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:28.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.571192 22927 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:46:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.576857 22927 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:46:28.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.577145 22927 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:46:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.577456 22927 ts_tablet_manager.cc:610] Registered 0 tablets
07:46:28.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.578045 22927 ts_tablet_manager.cc:589] Time spent register tablets: real 0.001s user 0.000s sys 0.000s
07:46:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.744247 22927 rpc_server.cc:307] RPC server started. Bound to: 127.22.29.129:39623
07:46:28.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.744313 23075 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.29.129:39623 every 8 connection(s)
07:46:28.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.747694 22927 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/data/info.pb
07:46:28.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.756011 22646 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 22927
07:46:28.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.756839 22646 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-0/wal/instance
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.781916 22646 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/wal
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/data
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/logs
07:46:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/data/info.pb
07:46:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:28.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.130:0
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.29.130
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.130
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:28.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:46:28.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.824545 23077 heartbeater.cc:347] Connected to a master server at 127.22.29.189:42083
07:46:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.825032 23077 heartbeater.cc:464] Registering TS with master...
07:46:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.826112 23077 heartbeater.cc:511] Master 127.22.29.189:42083 requested a full tablet report, sending...
07:46:28.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.829653 22766 ts_manager.cc:194] Registered new tserver with Master: 3f1b312598fe488198a0e4305edb5c01 (127.22.29.129:39623)
07:46:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.833925 23078 heartbeater.cc:347] Connected to a master server at 127.22.29.190:34791
07:46:28.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.834240 23078 heartbeater.cc:464] Registering TS with master...
07:46:28.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.834992 23078 heartbeater.cc:511] Master 127.22.29.190:34791 requested a full tablet report, sending...
07:46:28.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.837659 22691 ts_manager.cc:194] Registered new tserver with Master: 3f1b312598fe488198a0e4305edb5c01 (127.22.29.129:39623)
07:46:28.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.840842 22691 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.29.129:45579
07:46:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.842417 23076 heartbeater.cc:347] Connected to a master server at 127.22.29.188:36219
07:46:28.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.842733 23076 heartbeater.cc:464] Registering TS with master...
07:46:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.843570 23076 heartbeater.cc:511] Master 127.22.29.188:36219 requested a full tablet report, sending...
07:46:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:28.846617 22845 ts_manager.cc:194] Registered new tserver with Master: 3f1b312598fe488198a0e4305edb5c01 (127.22.29.129:39623)
07:46:29.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:29.130652 23086 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:29.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:29.131117 23086 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:29.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:29.131609 23086 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:29.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:29.161780 23086 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:29.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:29.162724 23086 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.29.130
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:29.196007 23086 tablet_server_runner.cc:78] Tablet server non-default flags:
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/data
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/wal
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.130:0
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/data/info.pb
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.130
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.29.130
07:46:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/logs
07:46:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:46:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:29.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:29.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:29.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:29.197520 23086 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:29.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:29.199509 23086 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:29.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:29.222540 23096 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:29.845811 23078 heartbeater.cc:503] Master 127.22.29.190:34791 was elected leader, sending a full tablet report...
07:46:30.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:29.222546 23094 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:30.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:29.226333 23093 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:30.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:30.527789 23095 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1305 milliseconds
07:46:30.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.527894 23086 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:46:30.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.529106 23086 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:30.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.532146 23086 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:30.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.533605 23086 hybrid_clock.cc:648] HybridClock initialized: now 1790235990533562 us; error 66 us; skew 500 ppm
07:46:30.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.534439 23086 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.541510 23086 webserver.cc:460] Webserver started at http://127.22.29.130:42277/ using document root <none> and password file <none>
07:46:30.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.542747 23086 fs_manager.cc:359] Metadata directory not provided
07:46:30.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.542980 23086 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:30.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.543687 23086 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:30.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.548666 23086 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/data/instance:
07:46:30.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40adeeaa9b0a47caaf6e5c1d0443f354"
07:46:30.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:30 on dist-test-slave-01fg"
07:46:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.549831 23086 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/wal/instance:
07:46:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40adeeaa9b0a47caaf6e5c1d0443f354"
07:46:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:30 on dist-test-slave-01fg"
07:46:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.558694 23086 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
07:46:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.564626 23107 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.565845 23086 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
07:46:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.566286 23086 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/wal
07:46:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40adeeaa9b0a47caaf6e5c1d0443f354"
07:46:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:30 on dist-test-slave-01fg"
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.566659 23086 fs_report.cc:383] FS layout report
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/wal
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/wal
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/data/data
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:30.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:30.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:30.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:30.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:30.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.612277 23086 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.613850 23086 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:30.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.614387 23086 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:30.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.617058 23086 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:46:30.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.622123 23086 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:46:30.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.622368 23086 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:46:30.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.622735 23086 ts_tablet_manager.cc:610] Registered 0 tablets
07:46:30.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.622932 23086 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:46:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.774746 23086 rpc_server.cc:307] RPC server started. Bound to: 127.22.29.130:42121
07:46:30.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.774840 23219 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.29.130:42121 every 8 connection(s)
07:46:30.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.777180 23086 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/data/info.pb
07:46:30.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.780918 22646 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 23086
07:46:30.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.781755 22646 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-1/wal/instance
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.798079 22646 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/wal
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/data
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/logs
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/data/info.pb
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:30.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:46:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.131:0
07:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.29.131
07:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.131
07:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:46:30.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.847105 23220 heartbeater.cc:347] Connected to a master server at 127.22.29.188:36219
07:46:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.847805 23222 heartbeater.cc:347] Connected to a master server at 127.22.29.190:34791
07:46:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.848100 23220 heartbeater.cc:464] Registering TS with master...
07:46:30.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.848232 23222 heartbeater.cc:464] Registering TS with master...
07:46:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.849414 23220 heartbeater.cc:511] Master 127.22.29.188:36219 requested a full tablet report, sending...
07:46:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.850308 23222 heartbeater.cc:511] Master 127.22.29.190:34791 requested a full tablet report, sending...
07:46:30.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.853360 22690 ts_manager.cc:194] Registered new tserver with Master: 40adeeaa9b0a47caaf6e5c1d0443f354 (127.22.29.130:42121)
07:46:30.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.853868 22845 ts_manager.cc:194] Registered new tserver with Master: 40adeeaa9b0a47caaf6e5c1d0443f354 (127.22.29.130:42121)
07:46:30.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.855919 22690 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.29.130:36815
07:46:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.862430 23221 heartbeater.cc:347] Connected to a master server at 127.22.29.189:42083
07:46:30.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.862812 23221 heartbeater.cc:464] Registering TS with master...
07:46:30.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.863549 23221 heartbeater.cc:511] Master 127.22.29.189:42083 requested a full tablet report, sending...
07:46:30.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:30.867050 22766 ts_manager.cc:194] Registered new tserver with Master: 40adeeaa9b0a47caaf6e5c1d0443f354 (127.22.29.130:42121)
07:46:31.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:31.156344 23227 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:31.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:31.156817 23227 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:31.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:31.157255 23227 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:31.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:31.189126 23227 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:31.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:31.189908 23227 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.29.131
07:46:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:31.224293 23227 tablet_server_runner.cc:78] Tablet server non-default flags:
07:46:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.29.148:44299
07:46:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/data
07:46:31.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/wal
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.29.131:0
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/data/info.pb
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.29.131
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.29.190:34791,127.22.29.189:42083,127.22.29.188:36219
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.29.131
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/logs
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:31.225605 23227 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:31.227082 23227 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:31.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:31.248117 23236 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:31.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:31.860299 23222 heartbeater.cc:503] Master 127.22.29.190:34791 was elected leader, sending a full tablet report...
07:46:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:31.248123 23239 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:31.248587 23237 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:32.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:32.535502 23238 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1287 milliseconds
07:46:32.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.535610 23227 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:46:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.536852 23227 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.539516 23227 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.540930 23227 hybrid_clock.cc:648] HybridClock initialized: now 1790235992540905 us; error 51 us; skew 500 ppm
07:46:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.541642 23227 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.548760 23227 webserver.cc:460] Webserver started at http://127.22.29.131:46025/ using document root <none> and password file <none>
07:46:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.549604 23227 fs_manager.cc:359] Metadata directory not provided
07:46:32.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.549782 23227 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:32.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.550158 23227 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.555012 23227 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/data/instance:
07:46:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e8352d5fa26448782138f46991734d9"
07:46:32.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:32 on dist-test-slave-01fg"
07:46:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.556689 23227 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/wal/instance:
07:46:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e8352d5fa26448782138f46991734d9"
07:46:32.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:32 on dist-test-slave-01fg"
07:46:32.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.564409 23227 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
07:46:32.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.569998 23247 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:32.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.570922 23227 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
07:46:32.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.571278 23227 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/wal
07:46:32.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e8352d5fa26448782138f46991734d9"
07:46:32.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:32 on dist-test-slave-01fg"
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.571664 23227 fs_report.cc:383] FS layout report
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/wal
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/wal
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/data/data
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:32.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:32.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.609304 23227 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:32.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.610764 23227 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:32.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.611233 23227 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:32.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.613757 23227 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:46:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.617782 23227 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:46:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.618018 23227 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:46:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.618322 23227 ts_tablet_manager.cc:610] Registered 0 tablets
07:46:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.618503 23227 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:46:32.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.766635 23227 rpc_server.cc:307] RPC server started. Bound to: 127.22.29.131:40757
07:46:32.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.766721 23359 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.29.131:40757 every 8 connection(s)
07:46:32.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.769071 23227 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/data/info.pb
07:46:32.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.777041 22646 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 23227
07:46:32.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.777539 22646 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster4334630035400341589/ts-2/wal/instance
07:46:32.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.834547 23360 heartbeater.cc:347] Connected to a master server at 127.22.29.188:36219
07:46:32.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.834988 23360 heartbeater.cc:464] Registering TS with master...
07:46:32.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.836040 23360 heartbeater.cc:511] Master 127.22.29.188:36219 requested a full tablet report, sending...
07:46:32.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.838692 22844 ts_manager.cc:194] Registered new tserver with Master: 5e8352d5fa26448782138f46991734d9 (127.22.29.131:40757)
07:46:32.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.840251 23361 heartbeater.cc:347] Connected to a master server at 127.22.29.189:42083
07:46:32.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.840581 23361 heartbeater.cc:464] Registering TS with master...
07:46:32.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.841281 23361 heartbeater.cc:511] Master 127.22.29.189:42083 requested a full tablet report, sending...
07:46:32.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.844282 22765 ts_manager.cc:194] Registered new tserver with Master: 5e8352d5fa26448782138f46991734d9 (127.22.29.131:40757)
07:46:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.846863 23362 heartbeater.cc:347] Connected to a master server at 127.22.29.190:34791
07:46:32.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.847158 23362 heartbeater.cc:464] Registering TS with master...
07:46:32.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.847985 23362 heartbeater.cc:511] Master 127.22.29.190:34791 requested a full tablet report, sending...
07:46:32.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.849489 22690 ts_manager.cc:194] Registered new tserver with Master: 5e8352d5fa26448782138f46991734d9 (127.22.29.131:40757)
07:46:32.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.850837 22690 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.22.29.131:47391
07:46:32.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:32.868397 22646 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:46:32.880 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:46:32.881 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:46:32.891 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.29.190"
port: 34791
}
bound_http_address {
host: "127.22.29.190"
port: 41529
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.29.189"
port: 42083
}
bound_http_address {
host: "127.22.29.189"
port: 38711
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.29.188"
port: 36219
}
bound_http_address {
host: "127.22.29.188"
port: 42199
}
}
}
07:46:32.896 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:46:32.898 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.29.129"
port: 39623
}
bound_http_address {
host: "127.22.29.129"
port: 43561
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.29.130"
port: 42121
}
bound_http_address {
host: "127.22.29.130"
port: 42277
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.29.131"
port: 40757
}
bound_http_address {
host: "127.22.29.131"
port: 46025
}
}
}
07:46:32.898 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:46:33.080 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:46:33.192 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task9m7ByE/test-tmp
07:46:33.192 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:46:33.193 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task9m7ByE/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:46:33.197 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:46:33.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:33.547 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:46:33.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:33.550513 23371 mini_oidc.cc:152] Starting JWKS server
07:46:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:33.554418 23371 webserver.cc:460] Webserver started at http://127.0.0.1:39607/ using document root <none> and password file <none>
07:46:33.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:33.554808 23371 mini_oidc.cc:189] Starting OIDC Discovery server
07:46:33.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:33.556454 23371 webserver.cc:460] Webserver started at http://127.0.0.1:40561/ using document root <none> and password file <none>
07:46:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:46:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/principal' for realm 'KRBTEST.COM',
07:46:33.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:46:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:33 dist-test-slave-01fg krb5kdc[23383](info): setting up network...
07:46:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:46:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:33 dist-test-slave-01fg krb5kdc[23383](info): set up 2 sockets
07:46:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:33 dist-test-slave-01fg krb5kdc[23383](info): commencing operation
07:46:33.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:46:33.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:33.855916 23362 heartbeater.cc:503] Master 127.22.29.190:34791 was elected leader, sending a full tablet report...
07:46:34.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:34.683070 23371 mini_kdc.cc:121] Time spent starting KDC: real 1.103s user 0.005s sys 0.015s
07:46:34.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:46:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:46:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:46:34.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:46:34.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:46:34.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:46:34.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:46:34.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:46:34.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:46:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:46:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/test-user.keytab.
07:46:34.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/test-user.keytab.
07:46:34.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:34 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235994, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:34.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:46:34.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:46:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:46:34.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:46:34Z Disabled control of system clock
07:46:34.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.210.254@KRBTEST.COM; defaulting to no policy
07:46:34.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:34.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.210.254@KRBTEST.COM" created.
07:46:34.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:34.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.210.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.254.keytab.
07:46:34.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.210.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.254.keytab.
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:34.880199 23371 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:34.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/logs
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/info.pb
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.254:43329
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.254
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.254.keytab
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.254
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:34.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:35.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.186615 23398 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:35.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.187254 23398 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:35.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.188194 23398 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.219241 23398 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.219545 23398 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.219972 23398 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.220374 23398 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.220696 23398 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:35.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.220999 23398 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:35.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.221261 23398 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39607/jwks.json
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:35.259014 23398 master_runner.cc:386] Master server non-default flags:
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.254.keytab
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.254
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.254:43329
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/info.pb
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.254
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/logs
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:35.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:35.260519 23398 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:35.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:35.261559 23398 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:35.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.283627 23407 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:36.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:35.285030 23398 server_base.cc:1034] running on GCE node
07:46:36.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.287657 23404 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:36.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:35.283665 23405 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:36.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.510843 23398 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:36.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.513685 23398 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:36.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.515208 23398 hybrid_clock.cc:648] HybridClock initialized: now 1790235996515187 us; error 34 us; skew 500 ppm
07:46:36.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.516131 23398 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:36 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235996, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:36.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.522101 23398 init.cc:377] Logged in from keytab as kudu/127.22.210.254@KRBTEST.COM (short username kudu)
07:46:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.526979 23398 webserver.cc:460] Webserver started at http://127.22.210.254:42637/ using document root <none> and password file <none>
07:46:36.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.527859 23398 fs_manager.cc:359] Metadata directory not provided
07:46:36.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.528033 23398 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:36.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.528389 23398 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.533272 23398 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/instance:
07:46:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b07c29e584df4e9cb29ff3bcad70868a"
07:46:36.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:36 on dist-test-slave-01fg"
07:46:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.534242 23398 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal/instance:
07:46:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b07c29e584df4e9cb29ff3bcad70868a"
07:46:36.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:36 on dist-test-slave-01fg"
07:46:36.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.541337 23398 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
07:46:36.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.547286 23415 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:36.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.548389 23398 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:46:36.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.548640 23398 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:36.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b07c29e584df4e9cb29ff3bcad70868a"
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:36 on dist-test-slave-01fg"
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.548899 23398 fs_report.cc:383] FS layout report
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/data
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:36.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.588189 23398 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:36.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.594456 23398 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:36.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.594960 23398 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.675346 23398 rpc_server.cc:307] RPC server started. Bound to: 127.22.210.254:43329
07:46:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.675451 23467 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.210.254:43329 every 8 connection(s)
07:46:36.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.678244 23398 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/info.pb
07:46:36.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.683060 23371 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 23398
07:46:36.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.683615 23371 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal/instance
07:46:36.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.683811 23468 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.701072 23468 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } has no permanent_uuid. Determining permanent_uuid...
07:46:36.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.210.253@KRBTEST.COM; defaulting to no policy
07:46:36.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.210.253@KRBTEST.COM" created.
07:46:36.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:36 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235996, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.254@KRBTEST.COM for kudu/127.22.210.254@KRBTEST.COM
07:46:36.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:36.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.210.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.253.keytab.
07:46:36.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.210.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.253.keytab.
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.733258 23371 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/logs
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/info.pb
07:46:36.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.253:41979
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.253
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.253.keytab
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.253
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:36.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.742270 23468 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } has no permanent_uuid. Determining permanent_uuid...
07:46:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:36.745047 23418 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:36.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:36.749644 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.253:41979: Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:36.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.799177 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } attempt: 1
07:46:36.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:36.803247 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.253:41979: Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:36.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.871809 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } attempt: 2
07:46:36.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:36.875283 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.253:41979: Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:36.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:36.967181 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } attempt: 3
07:46:36.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:36.971912 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.253:41979: Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:37.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.048113 23474 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:37.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.048612 23474 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:37.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.049154 23474 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.080164 23474 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.080372 23474 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.080636 23474 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:37.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.080828 23474 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.081041 23474 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.081228 23474 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:37.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.081377 23474 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39607/jwks.json
07:46:37.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:37.115343 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } attempt: 4
07:46:37.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:37.118034 23474 master_runner.cc:386] Master server non-default flags:
07:46:37.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:37.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:37.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.253.keytab
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.253
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.253:41979
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/info.pb
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.253
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/logs
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.119351 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.253:41979: Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:37.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:37.119968 23474 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:37.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:37.120862 23474 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:37.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.138449 23482 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:37.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:37.418813 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } attempt: 5
07:46:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.423811 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.253:41979: Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:37.971434 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } attempt: 6
07:46:37.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.979328 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.253:41979: Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.138638 23484 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:38.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:37.138453 23481 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:38.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.365103 23483 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:46:38.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.365242 23474 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:46:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.370626 23474 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:38.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.373121 23474 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:38.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.374486 23474 hybrid_clock.cc:648] HybridClock initialized: now 1790235998374461 us; error 35 us; skew 500 ppm
07:46:38.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.375294 23474 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:38.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:38 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235998, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:38.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.381230 23474 init.cc:377] Logged in from keytab as kudu/127.22.210.253@KRBTEST.COM (short username kudu)
07:46:38.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.386795 23474 webserver.cc:460] Webserver started at http://127.22.210.253:36199/ using document root <none> and password file <none>
07:46:38.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.387733 23474 fs_manager.cc:359] Metadata directory not provided
07:46:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.387959 23474 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:38.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.388437 23474 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.393262 23474 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/instance:
07:46:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbd85d9716b04672bc5e495bdfc4111e"
07:46:38.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:38 on dist-test-slave-01fg"
07:46:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.394738 23474 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal/instance:
07:46:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbd85d9716b04672bc5e495bdfc4111e"
07:46:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:38 on dist-test-slave-01fg"
07:46:38.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.402441 23474 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.010s sys 0.000s
07:46:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.408378 23494 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.409293 23474 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:46:38.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.409626 23474 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbd85d9716b04672bc5e495bdfc4111e"
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:38 on dist-test-slave-01fg"
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.409970 23474 fs_report.cc:383] FS layout report
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/data
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:38.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:38.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:38.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.452268 23474 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.458211 23474 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:38.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.458715 23474 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.542551 23474 rpc_server.cc:307] RPC server started. Bound to: 127.22.210.253:41979
07:46:38.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.542621 23545 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.210.253:41979 every 8 connection(s)
07:46:38.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.545608 23474 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/info.pb
07:46:38.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.553285 23371 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 23474
07:46:38.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.553404 23546 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:38.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.553792 23371 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal/instance
07:46:38.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.568284 23546 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } has no permanent_uuid. Determining permanent_uuid...
07:46:38.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.210.252@KRBTEST.COM; defaulting to no policy
07:46:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:38.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.210.252@KRBTEST.COM" created.
07:46:38.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:38 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235998, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.253@KRBTEST.COM for kudu/127.22.210.254@KRBTEST.COM
07:46:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.210.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.252.keytab.
07:46:38.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.210.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.252.keytab.
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.603317 23371 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:38.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/logs
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/info.pb
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.252:34475
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.252
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.252.keytab
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.252
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:38.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:38.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:38.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.609861 23546 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } has no permanent_uuid. Determining permanent_uuid...
07:46:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:38 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235998, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.253@KRBTEST.COM for kudu/127.22.210.253@KRBTEST.COM
07:46:38.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.650665 23546 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } has no permanent_uuid. Determining permanent_uuid...
07:46:38.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.652971 23495 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:38.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.657661 23546 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.707165 23546 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 1
07:46:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.711383 23546 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:38.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.779999 23546 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 2
07:46:38.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.784387 23546 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:38.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:38.876263 23546 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 3
07:46:38.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.882311 23546 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:38.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.936949 23552 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:38.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.937582 23552 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:38.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.938155 23552 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.971890 23552 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.972118 23552 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.972371 23552 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.972560 23552 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:38.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.972757 23552 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.972936 23552 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:38.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:38.973074 23552 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39607/jwks.json
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.010054 23552 master_runner.cc:386] Master server non-default flags:
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.252.keytab
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.252
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.252:34475
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:39.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/info.pb
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.252
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/logs
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.011251 23552 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.012181 23552 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:39.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.025701 23546 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 4
07:46:39.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.029752 23546 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:39.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.033030 23560 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:39.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.039945 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } attempt: 7
07:46:39.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:39 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235996, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.254@KRBTEST.COM for kudu/127.22.210.253@KRBTEST.COM
07:46:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.069267 23468 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } has no permanent_uuid. Determining permanent_uuid...
07:46:39.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.075121 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:39.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.133759 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 1
07:46:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.137557 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:39.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.218968 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 2
07:46:39.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.222856 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:39.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.308238 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 3
07:46:39.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.313019 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:39.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.329138 23546 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 5
07:46:39.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.333149 23546 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.453440 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 4
07:46:39.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.457599 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.740981 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 5
07:46:39.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.746038 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.880759 23546 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 6
07:46:39.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.898536 23546 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:40.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.033643 23561 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:40.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:39.034332 23563 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:40.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:39.034771 23552 server_base.cc:1034] running on GCE node
07:46:40.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.283331 23552 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:40.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.286580 23552 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:40.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.287997 23552 hybrid_clock.cc:648] HybridClock initialized: now 1790236000287956 us; error 58 us; skew 500 ppm
07:46:40.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.288829 23552 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236000, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.294665 23552 init.cc:377] Logged in from keytab as kudu/127.22.210.252@KRBTEST.COM (short username kudu)
07:46:40.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.298573 23468 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } attempt: 6
07:46:40.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.302014 23552 webserver.cc:460] Webserver started at http://127.22.210.252:34569/ using document root <none> and password file <none>
07:46:40.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.303479 23552 fs_manager.cc:359] Metadata directory not provided
07:46:40.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.303740 23552 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.304286 23552 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:40.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:40.305977 23468 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.22.210.252:34475: Network error: Client connection negotiation failed: client connection to 127.22.210.252:34475: connect: Connection refused (error 111)
07:46:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.309834 23552 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/instance:
07:46:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c556ba6392324c4b8ea582c12ad482aa"
07:46:40.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:40 on dist-test-slave-01fg"
07:46:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.310950 23552 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal/instance:
07:46:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c556ba6392324c4b8ea582c12ad482aa"
07:46:40.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:40 on dist-test-slave-01fg"
07:46:40.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.318535 23552 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
07:46:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.323978 23579 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.324996 23552 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
07:46:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.325330 23552 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c556ba6392324c4b8ea582c12ad482aa"
07:46:40.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:40 on dist-test-slave-01fg"
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.325649 23552 fs_report.cc:383] FS layout report
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/data
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:40.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:40.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:40.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:40.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:40.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.374015 23552 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:40.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.380327 23552 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:40.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.380857 23552 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.461225 23552 rpc_server.cc:307] RPC server started. Bound to: 127.22.210.252:34475
07:46:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.461314 23631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.210.252:34475 every 8 connection(s)
07:46:40.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.463804 23552 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/info.pb
07:46:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.468940 23371 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 23552
07:46:40.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.469362 23371 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal/instance
07:46:40.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.469899 23632 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:40.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.483902 23632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } has no permanent_uuid. Determining permanent_uuid...
07:46:40.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235994, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.210.252@KRBTEST.COM
07:46:40.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236000, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.252@KRBTEST.COM for kudu/127.22.210.254@KRBTEST.COM
07:46:40.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.521586 23632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } has no permanent_uuid. Determining permanent_uuid...
07:46:40.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236000, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.252@KRBTEST.COM for kudu/127.22.210.253@KRBTEST.COM
07:46:40.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235994, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.210.253@KRBTEST.COM
07:46:40.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.556439 23632 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } has no permanent_uuid. Determining permanent_uuid...
07:46:40.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236000, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.252@KRBTEST.COM for kudu/127.22.210.252@KRBTEST.COM
07:46:40.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790235994, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.210.254@KRBTEST.COM
07:46:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.591589 23632 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: Bootstrap starting.
07:46:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.591717 23371 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:46:40.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.598835 23632 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: Neither blocks nor log segments found. Creating new log.
07:46:40.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.601432 23632 log.cc:824] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: Log is configured to *not* fsync() on all Append() calls
07:46:40.604 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:40.606 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:46:40.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.606709 23632 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: No bootstrap required, opened a new log
07:46:40.615 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:46:40.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:46:40.619 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.210.254"
port: 43329
}
bound_http_address {
host: "127.22.210.254"
port: 42637
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.210.253"
port: 41979
}
bound_http_address {
host: "127.22.210.253"
port: 36199
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.210.252"
port: 34475
}
bound_http_address {
host: "127.22.210.252"
port: 34569
}
}
}
07:46:40.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:46:40.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:40.620 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:46:40.622 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:46:40.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.636248 23632 raft_consensus.cc:348] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:40.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.637002 23632 raft_consensus.cc:374] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:46:40.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.637264 23632 raft_consensus.cc:729] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c556ba6392324c4b8ea582c12ad482aa, State: Initialized, Role: FOLLOWER
07:46:40.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.638173 23632 consensus_queue.cc:260] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:40.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.641542 23642 sys_catalog.cc:455] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } } }
07:46:40.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.642554 23642 sys_catalog.cc:458] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [sys.catalog]: This master's current role is: FOLLOWER
07:46:40.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.643846 23632 sys_catalog.cc:564] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [sys.catalog]: configured and running, proceeding with master startup.
07:46:40.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236000, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:40.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:46:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.658836 23642 raft_consensus.cc:482] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.659456 23642 raft_consensus.cc:504] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:40.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.663794 23642 leader_election.cc:288] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329), bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979)
07:46:40.670 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:40.672 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc
07:46:40.679 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:46:40.680 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc
07:46:40.681 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:46:40.686 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:46:40.688 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:46:40.688 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:46:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:40.724444 23654 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:40.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:40.725351 23654 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:40.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.670514 23552 master_runner.cc:186] Error getting master registration for 127.22.210.254:43329: OK, instance_id { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" instance_seqno: 1790235996649192 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:46:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.748637 23552 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:40 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236000, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:46:40.775 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:40.775 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc
07:46:40.777 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:46:40.778 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.210.254:43329
07:46:40.781 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:46:40.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.781773 23371 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 23398
07:46:40.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:40.812585 23581 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.22.210.254:43329 (error 108)
07:46:40.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:40.818756 23581 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329): Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:46:40.820 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:40.820 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.210.253:41979
07:46:40.821 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:46:40.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.821671 23371 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 23474
07:46:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:40.855715 23582 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979): Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.856189 23582 leader_election.cc:302] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c556ba6392324c4b8ea582c12ad482aa; no voters: b07c29e584df4e9cb29ff3bcad70868a, bbd85d9716b04672bc5e495bdfc4111e
07:46:40.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.857021 23642 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:40.859 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:40.859 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.210.252:34475
07:46:40.859 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:46:40.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.859884 23371 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 23552
07:46:40.883 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:40.883 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.210.254:43329
07:46:40.887 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:40.889644 23371 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/logs
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/info.pb
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.254:43329
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.254
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42637
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.254.keytab
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.254
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:40.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:40.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:40.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:41.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.213018 23659 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:41.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.213555 23659 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.214126 23659 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:41.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.244778 23659 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:41.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.244993 23659 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:41.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.245252 23659 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:41.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.245442 23659 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:41.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.245712 23659 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.245914 23659 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:41.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.246060 23659 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39607/jwks.json
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:41.282686 23659 master_runner.cc:386] Master server non-default flags:
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.254.keytab
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.254
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.254:43329
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/info.pb
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.254
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42637
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/logs
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:41.283874 23659 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:41.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:41.284813 23659 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:41.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.304955 23667 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:41.307119 23659 server_base.cc:1034] running on GCE node
07:46:42.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.305981 23666 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:42.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:41.307807 23669 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:42.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.540683 23659 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:42.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.543174 23659 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:42.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.544564 23659 hybrid_clock.cc:648] HybridClock initialized: now 1790236002544549 us; error 51 us; skew 500 ppm
07:46:42.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.545387 23659 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:42.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:42 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236002, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:42.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.551388 23659 init.cc:377] Logged in from keytab as kudu/127.22.210.254@KRBTEST.COM (short username kudu)
07:46:42.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.559031 23659 webserver.cc:460] Webserver started at http://127.22.210.254:42637/ using document root <none> and password file <none>
07:46:42.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.560055 23659 fs_manager.cc:359] Metadata directory not provided
07:46:42.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.560360 23659 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:42.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.569360 23659 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.000s sys 0.004s
07:46:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.573755 23680 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.574782 23659 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
07:46:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.575158 23659 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b07c29e584df4e9cb29ff3bcad70868a"
07:46:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:36 on dist-test-slave-01fg"
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.577718 23659 fs_report.cc:383] FS layout report
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/data
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:42.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.631310 23659 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:42.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.640445 23659 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:42.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.640889 23659 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:42.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.716717 23659 rpc_server.cc:307] RPC server started. Bound to: 127.22.210.254:43329
07:46:42.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.716913 23731 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.210.254:43329 every 8 connection(s)
07:46:42.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.719588 23659 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/data/info.pb
07:46:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.722539 23371 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 23659
07:46:42.723 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:42.723 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.210.253:41979
07:46:42.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:46:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.726409 23371 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data
07:46:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/logs
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/info.pb
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:42.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.253:41979
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.253
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36199
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.253.keytab
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.253
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:42.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:42.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.734454 23732 sys_catalog.cc:263] Verifying existing consensus state
07:46:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 07:46:42.735195 23732 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:46:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.735586 23659 master.cc:537] Master@127.22.210.254:43329 shutting down...
07:46:42.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:42.807265 23659 master.cc:552] Master@127.22.210.254:43329 shutdown complete.
07:46:42.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:46:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.055200 23734 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:43.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.055908 23734 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:43.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.056619 23734 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:43.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.087425 23734 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.087702 23734 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.088181 23734 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:43.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.088528 23734 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.088878 23734 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.089232 23734 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:43.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.089521 23734 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39607/jwks.json
07:46:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:43.126823 23734 master_runner.cc:386] Master server non-default flags:
07:46:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data
07:46:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:43.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.253.keytab
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.253
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.253:41979
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/info.pb
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.253
07:46:43.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36199
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/logs
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:43.127985 23734 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:43.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:43.129101 23734 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:43.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.150043 23741 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:44.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.150049 23743 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:44.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:43.151047 23734 server_base.cc:1034] running on GCE node
07:46:44.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:43.150156 23740 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:44.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.405480 23734 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:44.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.408483 23734 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:44.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.409906 23734 hybrid_clock.cc:648] HybridClock initialized: now 1790236004409840 us; error 67 us; skew 500 ppm
07:46:44.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.410964 23734 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:44.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:44 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236004, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:44.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.427343 23734 init.cc:377] Logged in from keytab as kudu/127.22.210.253@KRBTEST.COM (short username kudu)
07:46:44.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.432590 23734 webserver.cc:460] Webserver started at http://127.22.210.253:36199/ using document root <none> and password file <none>
07:46:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.433497 23734 fs_manager.cc:359] Metadata directory not provided
07:46:44.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.433688 23734 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:44.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.441323 23734 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
07:46:44.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.445662 23753 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:44.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.446631 23734 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
07:46:44.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.446954 23734 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:44.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbd85d9716b04672bc5e495bdfc4111e"
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:38 on dist-test-slave-01fg"
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.449359 23734 fs_report.cc:383] FS layout report
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/data
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:44.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.498379 23734 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.504305 23734 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.504755 23734 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:44.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.585114 23734 rpc_server.cc:307] RPC server started. Bound to: 127.22.210.253:41979
07:46:44.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.585240 23804 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.210.253:41979 every 8 connection(s)
07:46:44.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.588835 23734 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/data/info.pb
07:46:44.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.597847 23371 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 23734
07:46:44.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:44.599 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.210.252:34475
07:46:44.600 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:46:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.601439 23371 external_mini_cluster.cc:1326] Running /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu
07:46:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/logs
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/info.pb
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.252:34475
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.252
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34569
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.252.keytab
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.252
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:44.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.604820 23805 sys_catalog.cc:263] Verifying existing consensus state
07:46:44.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260924 07:46:44.605767 23805 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:46:44.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.606284 23734 master.cc:537] Master@127.22.210.253:41979 shutting down...
07:46:44.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:44.678736 23734 master.cc:552] Master@127.22.210.253:41979 shutdown complete.
07:46:44.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:46:44.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.940655 23807 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:44.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.941164 23807 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:44.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.941694 23807 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.977149 23807 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.977358 23807 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.977607 23807 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:44.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.977813 23807 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.977990 23807 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.978193 23807 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:44.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:44.978507 23807 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39607/jwks.json
07:46:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:45.017287 23807 master_runner.cc:386] Master server non-default flags:
07:46:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.210.212:42653
07:46:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/krb5kdc/kudu_127.22.210.252.keytab
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.210.252
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.210.252:34475
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39607/jwks.json
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/info.pb
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.210.252
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34569
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/logs
07:46:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 24 Sep 2026 06:42:32 UTC on ccycloud.quasar-lgarko.root.comops.site
07:46:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4669406
07:46:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:45.018735 23807 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:45.020092 23807 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:45.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:45.037336 23813 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:46.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:45.038157 23814 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:46.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:45.038060 23816 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:46.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:45.038872 23807 server_base.cc:1034] running on GCE node
07:46:46.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.289566 23807 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:46.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.292273 23807 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:46.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.293644 23807 hybrid_clock.cc:648] HybridClock initialized: now 1790236006293605 us; error 54 us; skew 500 ppm
07:46:46.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.294425 23807 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:46.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:46 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236006, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:46.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.300666 23807 init.cc:377] Logged in from keytab as kudu/127.22.210.252@KRBTEST.COM (short username kudu)
07:46:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.306195 23807 webserver.cc:460] Webserver started at http://127.22.210.252:34569/ using document root <none> and password file <none>
07:46:46.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.307309 23807 fs_manager.cc:359] Metadata directory not provided
07:46:46.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.307595 23807 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:46.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.318250 23807 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
07:46:46.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.323585 23824 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.324782 23807 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
07:46:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.325165 23807 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data,/tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c556ba6392324c4b8ea582c12ad482aa"
07:46:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-24 07:46:40 on dist-test-slave-01fg"
07:46:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.328302 23807 fs_report.cc:383] FS layout report
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/wal
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/data
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:46.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.368160 23807 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:46.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.374464 23807 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:46.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.374962 23807 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:46.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.446112 23807 rpc_server.cc:307] RPC server started. Bound to: 127.22.210.252:34475
07:46:46.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.446383 23875 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.22.210.252:34475 every 8 connection(s)
07:46:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.449023 23807 server_base.cc:1166] Dumped server information to /tmp/dist-test-task9m7ByE/test-tmp/mini-kudu-cluster7126367953981819581/master-2/data/info.pb
07:46:46.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.452950 23371 external_mini_cluster.cc:1388] Started /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu as pid 23807
07:46:46.453 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:46.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.462543 23876 sys_catalog.cc:263] Verifying existing consensus state
07:46:46.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.470268 23876 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: Bootstrap starting.
07:46:46.511 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:46.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.513989 23876 log.cc:824] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: Log is configured to *not* fsync() on all Append() calls
07:46:46.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.519025 23876 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:46:46.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.519870 23876 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: Bootstrap complete.
07:46:46.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.550427 23876 raft_consensus.cc:348] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.553748 23876 raft_consensus.cc:374] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:46:46.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.554067 23876 raft_consensus.cc:729] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c556ba6392324c4b8ea582c12ad482aa, State: Initialized, Role: FOLLOWER
07:46:46.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.554988 23876 consensus_queue.cc:260] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:46.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.571672 23880 sys_catalog.cc:455] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } } }
07:46:46.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.572794 23880 sys_catalog.cc:458] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [sys.catalog]: This master's current role is: FOLLOWER
07:46:46.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.573576 23876 sys_catalog.cc:564] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [sys.catalog]: configured and running, proceeding with master startup.
07:46:46.609 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:46.611 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.210.252:34475(127.22.210.252:34475)] connecting to peer
07:46:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:46.630045 23830 tablet.cc:2369] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:46:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:46.630402 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:46.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:46.630683 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:46.664 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:46.664 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:46.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.625751 23807 master_runner.cc:186] Error getting master registration for 127.22.210.254:43329: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111),
07:46:46.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.669279 23807 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:46.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:46.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:46.672 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:46.674 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:46.675 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.210.252:34475(127.22.210.252:34475)] Successfully connected to peer
07:46:46.776 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:46:46.787 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.22.210.252
07:46:46.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.915519 23880 raft_consensus.cc:482] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:46.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.916057 23880 raft_consensus.cc:504] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:46.921466 23826 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:46:46.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.929347 23880 leader_election.cc:288] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329), bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979)
07:46:46.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:46.940407 23827 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979): Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:46.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:46.948205 23826 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329): Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:46:46.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.948782 23826 leader_election.cc:302] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c556ba6392324c4b8ea582c12ad482aa; no voters: b07c29e584df4e9cb29ff3bcad70868a, bbd85d9716b04672bc5e495bdfc4111e
07:46:46.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:46.949792 23880 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:47.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:47 dist-test-slave-01fg krb5kdc[23383](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236000, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.210.252@KRBTEST.COM
07:46:47.062 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x700a035e, L:/127.0.0.1:52694 - R:/127.22.210.252:34475] using SASL/GSSAPI
07:46:47.078 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.100 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.102 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.105 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.105 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.106 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.108 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.108 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.119 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.120 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.121 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.121 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.122 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.126 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.126 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.126 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.127 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.139 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.140 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.141 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.141 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.142 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.144 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.144 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.158 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.159 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.161 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.161 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.162 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.163 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.164 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.166 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.199 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.200 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.201 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.201 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.202 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.207 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.207 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.207 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.208 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.219 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.221 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.221 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.222 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.223 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.226 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.226 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.227 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.229 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.339 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.341 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.343 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.343 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.344 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.345 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.346 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.346 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.348 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.519 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.523 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.523 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.525 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.525 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.533 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:47.633949 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:47.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:47.634235 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:47.739 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:47.741 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:47.742 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.742 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.743 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:47.748 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.748 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:47.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:48.637199 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:48.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:48.637477 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:48.659 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:48.660 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:48.661 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:48.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:48.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:48.662 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:48.662 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:48.662 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:48.665 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:48.999 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:49.000 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:49.001 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:49.002 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:49.002 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:49.002 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:49.003 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:49.003 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:49.005 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:49.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:49.146490 23901 raft_consensus.cc:482] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:49.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:49.146833 23901 raft_consensus.cc:504] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:49.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:49.148365 23901 leader_election.cc:288] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329), bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979)
07:46:49.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:49.154610 23826 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329): Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:46:49.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:49.155083 23827 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979): Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:49.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:49.155443 23827 leader_election.cc:302] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c556ba6392324c4b8ea582c12ad482aa; no voters: b07c29e584df4e9cb29ff3bcad70868a, bbd85d9716b04672bc5e495bdfc4111e
07:46:49.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:49.156030 23901 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:49.640290 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:49.640618 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:50.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:50.625044 22876 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3940 after lost signal to thread 22808
07:46:50.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:50.625646 22876 debug-util.cc:398] Leaking SignalData structure 0x7b08000af600 after lost signal to thread 22879
07:46:50.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:50.644834 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:50.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:50.645161 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:50.814363 23907 raft_consensus.cc:482] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:50.814824 23907 raft_consensus.cc:504] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:50.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:50.816890 23907 leader_election.cc:288] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329), bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979)
07:46:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:50.822471 23827 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979): Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:50.823181 23826 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329): Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:46:50.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:50.823516 23826 leader_election.cc:302] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c556ba6392324c4b8ea582c12ad482aa; no voters: b07c29e584df4e9cb29ff3bcad70868a, bbd85d9716b04672bc5e495bdfc4111e
07:46:50.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:50.824144 23907 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:51.649281 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:51.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:51.649610 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:51.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:51.671820 23807 master_runner.cc:186] Error getting master registration for 127.22.210.254:43329: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111),
07:46:51.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:51.672304 23807 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:52.518 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:52.520 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:52.521 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:52.521 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:52.521 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:52.522 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:52.522 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:52.523 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:52.524 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:52.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:52.652803 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:52.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:52.653039 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:53.655822 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:53.656059 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:53.751662 23915 raft_consensus.cc:482] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:53.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:53.751986 23915 raft_consensus.cc:504] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:53.753477 23915 leader_election.cc:288] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329), bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979)
07:46:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:53.755530 23826 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:46:53.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:53.758360 23827 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979): Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:53.758756 23826 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329): Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:46:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:53.759053 23826 leader_election.cc:302] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c556ba6392324c4b8ea582c12ad482aa; no voters: b07c29e584df4e9cb29ff3bcad70868a, bbd85d9716b04672bc5e495bdfc4111e
07:46:53.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:53.759589 23915 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:54.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:54.124562 23072 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 22946
07:46:54.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:54.125627 23072 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7e80 after lost signal to thread 23075
07:46:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:54.658737 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:54.658985 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:55.663951 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:55.664367 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:55.899 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:55.900 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:55.902 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:55.902 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:55.902 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:55.903 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:55.903 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:55.903 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:55.906 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:46:56 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236016, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:56.304809 23820 init.cc:295] Successfully reacquired a new kerberos TGT
07:46:56.319 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:56.321 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:56.322 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:56.322 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:56.323 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:56.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:56.324 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:56.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:56.326 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:56.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:56.667697 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:56.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:56.667959 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:56.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:56.675859 23807 master_runner.cc:186] Error getting master registration for 127.22.210.254:43329: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111),
07:46:56.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:56.676275 23807 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:57.459 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:57.460 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:57.462 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:57.462 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:57.462 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:57.463 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:57.463 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:57.463 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:57.465 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:57.539 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:46:57.540 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:46:57.541 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:57.541 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:57.542 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:46:57.543 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:57.544 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:57.544 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:46:57.546 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:46:57.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:57.670769 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:57.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:57.671028 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:58.673702 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:58.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:58.673938 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:58.722816 23924 raft_consensus.cc:482] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:58.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:58.723320 23924 raft_consensus.cc:504] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:46:58.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:58.725252 23924 leader_election.cc:288] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329), bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979)
07:46:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:58.729195 23826 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329): Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:46:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:58.729681 23827 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979): Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:46:58.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:58.730093 23827 leader_election.cc:302] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c556ba6392324c4b8ea582c12ad482aa; no voters: b07c29e584df4e9cb29ff3bcad70868a, bbd85d9716b04672bc5e495bdfc4111e
07:46:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:46:58.730772 23924 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:59.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:59.676754 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:59.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:46:59.676981 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:00.219 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:00.222 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:00.223 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:00.223 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:00.223 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:00.225 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:00.225 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:00.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:00.227 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:00.459 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:00.460 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:00.462 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:00.462 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:00.463 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:00.463 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:00.463 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:00.463 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:00.465 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:00.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:00.679926 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:00.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:00.680181 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:01.678964 23807 master_runner.cc:186] Error getting master registration for 127.22.210.254:43329: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111),
07:47:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:01.679382 23807 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:47:01.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:01.682797 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:01.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:01.683071 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:02.619 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:02.620 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:02.621 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:02.622 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:02.622 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:02.622 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:02.622 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:02.623 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:02.625 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:02.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:02.686152 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:02.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:02.686455 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:03.579 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:03.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:03.580 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:03.580 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:03.580 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:03.582 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:03.582 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:03.582 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:03.584 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:03.689249 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:03.689486 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:03.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:03.748266 23935 raft_consensus.cc:482] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:47:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:03.748697 23935 raft_consensus.cc:504] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:47:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:03.750370 23935 leader_election.cc:288] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329), bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979)
07:47:03.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:03.752203 23826 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111) [suppressed 3 similar messages]
07:47:03.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:03.754838 23827 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979): Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:47:03.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:03.755319 23826 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329): Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:47:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:03.755695 23826 leader_election.cc:302] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c556ba6392324c4b8ea582c12ad482aa; no voters: b07c29e584df4e9cb29ff3bcad70868a, bbd85d9716b04672bc5e495bdfc4111e
07:47:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:03.756215 23935 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:47:04.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:04.692425 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:04.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:04.692677 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:04.940 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:04.940 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:04.941 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:04.941 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:04.941 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:04.942 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:04.942 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:04.942 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:04.945 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:05.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:05.695715 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:05.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:05.696022 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 24 07:47:06 dist-test-slave-01fg krb5kdc[23383](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790236026, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.210.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:47:06.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:06.307066 23820 init.cc:295] Successfully reacquired a new kerberos TGT
07:47:06.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:06.682340 23807 master_runner.cc:186] Error getting master registration for 127.22.210.254:43329: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111),
07:47:06.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:06.682709 23807 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:47:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:06.698441 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:06.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:06.698650 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:07.701252 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:07.701498 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:08.519 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:08.520 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:08.520 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:08.521 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:08.521 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:08.522 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:08.522 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:08.522 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:08.524 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:08.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:08.704538 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:08.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:08.704774 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:09.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:09.246847 23944 raft_consensus.cc:482] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:47:09.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:09.247324 23944 raft_consensus.cc:504] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b07c29e584df4e9cb29ff3bcad70868a" member_type: VOTER last_known_addr { host: "127.22.210.254" port: 43329 } } peers { permanent_uuid: "bbd85d9716b04672bc5e495bdfc4111e" member_type: VOTER last_known_addr { host: "127.22.210.253" port: 41979 } } peers { permanent_uuid: "c556ba6392324c4b8ea582c12ad482aa" member_type: VOTER last_known_addr { host: "127.22.210.252" port: 34475 } }
07:47:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:09.249428 23944 leader_election.cc:288] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329), bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979)
07:47:09.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:09.251319 23826 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111) [suppressed 1 similar messages]
07:47:09.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:09.254593 23827 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbd85d9716b04672bc5e495bdfc4111e (127.22.210.253:41979): Network error: Client connection negotiation failed: client connection to 127.22.210.253:41979: connect: Connection refused (error 111)
07:47:09.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:09.255435 23826 leader_election.cc:334] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b07c29e584df4e9cb29ff3bcad70868a (127.22.210.254:43329): Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111)
07:47:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:09.255769 23826 leader_election.cc:302] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c556ba6392324c4b8ea582c12ad482aa; no voters: b07c29e584df4e9cb29ff3bcad70868a, bbd85d9716b04672bc5e495bdfc4111e
07:47:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:09.256378 23944 raft_consensus.cc:2726] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:47:09.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:09.707729 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:09.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:09.708098 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:10.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:10.710855 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:10.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:10.711136 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:11.499 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:11.499 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:11.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:11.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:11.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:11.501 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:11.501 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:11.501 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:11.504 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:11.685272 23807 master_runner.cc:186] Error getting master registration for 127.22.210.254:43329: Network error: Client connection negotiation failed: client connection to 127.22.210.254:43329: connect: Connection refused (error 111),
07:47:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:11.685699 23807 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:47:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:11.713929 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:11.714216 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:12.299 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:12.299 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:12.300 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:12.300 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:12.300 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:12.301 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:12.301 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:12.301 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:12.303 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:12.717170 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:12.717454 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:13.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:13.720321 23893 catalog_manager.cc:1558] T 00000000000000000000000000000000 P c556ba6392324c4b8ea582c12ad482aa: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:47:13.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:13.720588 23893 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:47:13.878 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:13.879 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:13.880 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:13.880 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:13.880 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:13.881 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:13.881 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:13.881 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:13.883 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:14.519 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.254:43329(127.22.210.254:43329)] connecting to peer
07:47:14.520 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.22.210.253:41979(127.22.210.253:41979)] connecting to peer
07:47:14.520 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:14.520 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.210.254:43329(127.22.210.254:43329)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:14.520 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.254:43329: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329
07:47:14.521 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:14.521 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.253:41979(127.22.210.253:41979)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:14.522 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.210.253:41979: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
07:47:14.523 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475; will retry
07:47:14.528 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28066), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [208ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [612ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [646ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [648ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [659ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [666ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [679ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [689ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [699ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [739ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [744ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [746ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [761ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [766ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [881ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [884ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [886ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1064ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1068ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1071ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1071ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1289ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2196ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2203ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2536ms] refreshing cache from master, [2537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2540ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2541ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2543ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2543ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [6056ms] refreshing cache from master, [6056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [6057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [6058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [6059ms] too many traces: truncated at 100 traces, deferred=Deferred@1853102302(state=PENDING, result=null, callback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294), errback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294)))
07:47:14.532 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.210.252:34475(127.22.210.252:34475)] disconnecting while in state READY
07:47:14.533 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.210.254:43329(127.22.210.254:43329)] disconnecting while in state TERMINATED
07:47:14.533 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.210.253:41979(127.22.210.253:41979)] disconnecting while in state TERMINATED
07:47:14.538 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:47:14.540 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:47:14.541 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.210.252:34475(127.22.210.252:34475)] handling channelInactive
07:47:14.541 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.210.252:34475(127.22.210.252:34475)] cleaning up while in state READY due to: connection closed
07:47:14.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:14.544917 23371 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 23807
07:47:15.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:47:15Z chronyd exiting
07:47:16.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:16.166816 23072 debug-util.cc:398] Leaking SignalData structure 0x7b08000af220 after lost signal to thread 22946
07:47:16.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260924 07:47:16.167716 23072 debug-util.cc:398] Leaking SignalData structure 0x7b08000af140 after lost signal to thread 23075
07:47:16.786 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:47:16.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:16.787958 22646 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 22927
07:47:16.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:16.815789 22646 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 23086
07:47:16.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:16.841286 22646 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 23227
07:47:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:16.865489 22646 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 22658
07:47:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:16.896083 22646 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 22728
07:47:16.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260924 07:47:16.926823 22646 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task9m7ByE/build/tsan/bin/kudu with pid 22803
07:47:18.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-24T07:47:18Z chronyd exiting
07:47:19.110 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28066), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [208ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [612ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [646ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [648ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [659ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [666ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [679ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [689ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [699ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [739ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [744ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [746ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [761ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [766ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [881ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [884ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [886ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1064ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1068ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1071ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1071ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1289ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2196ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2203ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2536ms] refreshing cache from master, [2537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2540ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2541ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2543ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2543ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [6056ms] refreshing cache from master, [6056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [6057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [6058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [6059ms] too many traces: truncated at 100 traces, deferred=Deferred@1853102302(state=PENDING, result=null, callback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294), errback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.210.253:41979
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28066), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [208ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [612ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [646ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [648ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [659ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [666ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [679ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [689ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [699ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [739ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [744ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [746ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [761ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [766ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [881ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [884ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [886ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1064ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1068ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1071ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1071ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1289ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2196ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2203ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2536ms] refreshing cache from master, [2537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2540ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2541ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2543ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2543ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [6056ms] refreshing cache from master, [6056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [6057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [6058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [6059ms] too many traces: truncated at 100 traces, deferred=Deferred@1853102302(state=PENDING, result=null, callback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294), errback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.210.253:41979
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
07:47:19.886 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
07:47:19.886 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28066), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [208ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [612ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [646ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [648ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [659ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [666ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [679ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [689ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [699ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [739ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [744ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [746ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [761ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [766ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [881ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [884ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [886ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1064ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1068ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1071ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1071ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1289ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2196ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2203ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2536ms] refreshing cache from master, [2537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2540ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2541ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2543ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2543ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [6056ms] refreshing cache from master, [6056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [6057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [6058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [6059ms] too many traces: truncated at 100 traces, deferred=Deferred@1853102302(state=PENDING, result=null, callback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294), errback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.210.253:41979
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28066), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [147ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [149ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [208ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [612ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [617ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [637ms] refreshing cache from master, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [640ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [641ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [643ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [646ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [648ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [649ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [656ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [659ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [663ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [664ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [666ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [679ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [682ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [689ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [690ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [699ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [701ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [703ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [739ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [744ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [745ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [746ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [756ms] refreshing cache from master, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [761ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [764ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [766ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [877ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [880ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [881ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [884ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [886ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [886ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1056ms] refreshing cache from master, [1057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1062ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1063ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1064ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1068ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1071ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1071ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1276ms] refreshing cache from master, [1277ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [1278ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [1280ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [1280ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [1287ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [1289ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [1289ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2196ms] refreshing cache from master, [2197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2198ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2199ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2200ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2203ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2203ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2536ms] refreshing cache from master, [2537ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [2538ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [2539ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [2540ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.254:43329: Network error: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329, [2541ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.253:41979: Network error: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [2543ms] Sub RPC ConnectToMaster: received response from server master-127.22.210.252:34475: OK, [2543ms] delaying RPC due to: Service unavailable: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979, [6056ms] refreshing cache from master, [6056ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.254:43329, [6057ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.253:41979, [6058ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.210.252:34475, [6059ms] too many traces: truncated at 100 traces, deferred=Deferred@1853102302(state=PENDING, result=null, callback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294), errback=(continuation of Deferred@767480657 after retry RPC after error@194490187) -> (continuation of Deferred@822901983 after retry RPC after error@2138058952) -> (continuation of Deferred@613367852 after retry RPC after error@307514881) -> (continuation of Deferred@1080832528 after retry RPC after error@1046789285) -> (continuation of Deferred@1614021989 after retry RPC after error@1344301860) -> (continuation of Deferred@1912418178 after retry RPC after error@1793537164) -> (continuation of Deferred@708320152 after retry RPC after error@2031234224) -> (continuation of Deferred@1628151087 after retry RPC after error@849057494) -> (continuation of Deferred@1486204870 after retry RPC after error@805990344) -> (continuation of Deferred@223702531 after retry RPC after error@1670804569) -> (continuation of Deferred@369626226 after retry RPC after error@1964987732) -> (continuation of Deferred@1394346114 after retry RPC after error@66619908) -> (continuation of Deferred@1439002005 after retry RPC after error@1526675340) -> (continuation of Deferred@1762950217 after retry RPC after error@788227013) -> (continuation of Deferred@1353456202 after retry RPC after error@1934361732) -> (continuation of Deferred@972085392 after retry RPC after error@1281384736) -> (continuation of Deferred@2083409068 after retry RPC after error@1876238217) -> (continuation of Deferred@1356758715 after retry RPC after error@527989985) -> (continuation of Deferred@297601166 after retry RPC after error@1448309863) -> (continuation of Deferred@1977344944 after retry RPC after error@689440414) -> (continuation of Deferred@1343783499 after retry RPC after error@497362019) -> (continuation of Deferred@935956996 after retry RPC after error@1577864853) -> (continuation of Deferred@1242727877 after retry RPC after error@2116036312) -> (continuation of Deferred@84066986 after retry RPC after error@969301113) -> (continuation of Deferred@1738857001 after retry RPC after error@416102115) -> (continuation of Deferred@197763266 after retry RPC after error@6750294)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.22.210.254:43329,127.22.210.253:41979,127.22.210.252:34475) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.254:43329(127.22.210.254:43329): Connection refused: /127.22.210.254:43329,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.22.210.253:41979(127.22.210.253:41979): Connection refused: /127.22.210.253:41979
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.22.210.253:41979
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)