07:20:08.798 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:20:08.810 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNA4_Ab/test-tmp
07:20:08.814 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:20:08.817 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNA4_Ab/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:20:09.033 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:20:09.305 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:20:09.306 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:20:09.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.311688 17064 mini_oidc.cc:152] Starting JWKS server
07:20:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.317226 17064 webserver.cc:460] Webserver started at http://127.0.0.1:39411/ using document root <none> and password file <none>
07:20:09.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.317755 17064 mini_oidc.cc:189] Starting OIDC Discovery server
07:20:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.320188 17064 webserver.cc:460] Webserver started at http://127.0.0.1:45635/ using document root <none> and password file <none>
07:20:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:20:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:20:09.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:20:09Z Disabled control of system clock
07:20:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.378111 17064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:09.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/wal
07:20:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/data
07:20:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/logs
07:20:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/data/info.pb
07:20:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:09.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:09.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.62:41625
07:20:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.62
07:20:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:09.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:09.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json with env {}
07:20:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.706017 17081 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.706683 17081 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:09.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.707288 17081 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:09.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.738747 17081 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:09.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.739023 17081 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:09.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.739343 17081 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.739658 17081 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.740012 17081 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.740262 17081 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39411/jwks.json
07:20:09.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.782357 17081 master_runner.cc:386] Master server non-default flags:
07:20:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:09.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/data
07:20:09.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/wal
07:20:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.62:41625
07:20:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:09.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/data/info.pb
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.62
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:09.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/logs
07:20:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.784042 17081 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.785825 17081 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.808954 17088 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.808979 17093 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:09.808977 17087 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:09.811527 17081 server_base.cc:1034] running on GCE node
07:20:11.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.030272 17081 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:11.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.033234 17081 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.034588 17081 hybrid_clock.cc:648] HybridClock initialized: now 1786000811034592 us; error 82 us; skew 500 ppm
07:20:11.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.035719 17081 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:11.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.046855 17081 webserver.cc:460] Webserver started at http://127.16.170.62:37331/ using document root <none> and password file <none>
07:20:11.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.048175 17081 fs_manager.cc:359] Metadata directory not provided
07:20:11.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.048385 17081 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:11.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.048998 17081 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:11.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.055055 17081 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/data/instance:
07:20:11.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42058ffa4c06431aa32f479c7d506370"
07:20:11.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:11 on dist-test-slave-04bb"
07:20:11.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.056679 17081 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/wal/instance:
07:20:11.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42058ffa4c06431aa32f479c7d506370"
07:20:11.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:11 on dist-test-slave-04bb"
07:20:11.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.066242 17081 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
07:20:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.073704 17102 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:11.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.075067 17081 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
07:20:11.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.075492 17081 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/wal
07:20:11.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42058ffa4c06431aa32f479c7d506370"
07:20:11.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:11 on dist-test-slave-04bb"
07:20:11.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.076121 17081 fs_report.cc:383] FS layout report
07:20:11.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/wal
07:20:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/wal
07:20:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/data/data
07:20:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:11.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:11.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:11.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.122803 17081 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:11.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.124507 17081 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:11.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.125113 17081 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.207537 17081 rpc_server.cc:307] RPC server started. Bound to: 127.16.170.62:41625
07:20:11.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.207607 17171 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.170.62:41625 every 8 connection(s)
07:20:11.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.210922 17081 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/data/info.pb
07:20:11.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.213640 17064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 17081
07:20:11.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.214346 17064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-0/wal/instance
07:20:11.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.218357 17173 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.218760 17064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/wal
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/data
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/logs
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/data/info.pb
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:11.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.61:46189
07:20:11.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.61
07:20:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:11.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json with env {}
07:20:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.232106 17173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } has no permanent_uuid. Determining permanent_uuid...
07:20:11.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.267956 17173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } has no permanent_uuid. Determining permanent_uuid...
07:20:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.270203 17103 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.170.61:46189: connect: Connection refused (error 111)
07:20:11.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.274374 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.61:46189: Network error: Client connection negotiation failed: client connection to 127.16.170.61:46189: connect: Connection refused (error 111)
07:20:11.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.323953 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } attempt: 1
07:20:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.328776 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.61:46189: Network error: Client connection negotiation failed: client connection to 127.16.170.61:46189: connect: Connection refused (error 111)
07:20:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.397286 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } attempt: 2
07:20:11.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.401607 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.61:46189: Network error: Client connection negotiation failed: client connection to 127.16.170.61:46189: connect: Connection refused (error 111)
07:20:11.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.493098 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } attempt: 3
07:20:11.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.497063 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.61:46189: Network error: Client connection negotiation failed: client connection to 127.16.170.61:46189: connect: Connection refused (error 111)
07:20:11.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.545830 17176 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:11.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.546384 17176 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:11.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.546970 17176 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:11.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.578527 17176 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.578801 17176 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.579123 17176 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.579437 17176 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:11.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.579722 17176 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:11.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.580287 17176 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39411/jwks.json
07:20:11.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.639828 17176 master_runner.cc:386] Master server non-default flags:
07:20:11.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:11.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/data
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/wal
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:11.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.61:46189
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/data/info.pb
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.61
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:11.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/logs
07:20:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:11.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:11.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:11.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:11.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:11.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.641396 17176 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:11.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.643525 17176 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:11.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.647079 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } attempt: 4
07:20:11.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.652238 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.61:46189: Network error: Client connection negotiation failed: client connection to 127.16.170.61:46189: connect: Connection refused (error 111)
07:20:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.664713 17190 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.951689 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } attempt: 5
07:20:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.956236 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.61:46189: Network error: Client connection negotiation failed: client connection to 127.16.170.61:46189: connect: Connection refused (error 111)
07:20:12.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.503631 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } attempt: 6
07:20:12.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:12.511864 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.61:46189: Network error: Client connection negotiation failed: client connection to 127.16.170.61:46189: connect: Connection refused (error 111)
07:20:12.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:12.640187 17166 debug-util.cc:398] Leaking SignalData structure 0x7b0800072060 after lost signal to thread 17082
07:20:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:12.641031 17166 debug-util.cc:398] Leaking SignalData structure 0x7b08000727a0 after lost signal to thread 17171
07:20:12.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.665410 17189 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:11.668638 17176 server_base.cc:1034] running on GCE node
07:20:12.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:11.665633 17192 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:12.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.876629 17176 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:12.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.880694 17176 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.882228 17176 hybrid_clock.cc:648] HybridClock initialized: now 1786000812882155 us; error 89 us; skew 500 ppm
07:20:12.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.883457 17176 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.892994 17176 webserver.cc:460] Webserver started at http://127.16.170.61:44465/ using document root <none> and password file <none>
07:20:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.893990 17176 fs_manager.cc:359] Metadata directory not provided
07:20:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.894198 17176 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.894619 17176 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:12.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.900262 17176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/data/instance:
07:20:12.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a9f4b3b385344a6b325fddc1c0f608b"
07:20:12.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:12 on dist-test-slave-04bb"
07:20:12.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.901962 17176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/wal/instance:
07:20:12.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a9f4b3b385344a6b325fddc1c0f608b"
07:20:12.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:12 on dist-test-slave-04bb"
07:20:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.911041 17176 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
07:20:12.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.918139 17204 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:12.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.919456 17176 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
07:20:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.919780 17176 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/wal
07:20:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6a9f4b3b385344a6b325fddc1c0f608b"
07:20:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:12 on dist-test-slave-04bb"
07:20:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.920121 17176 fs_report.cc:383] FS layout report
07:20:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:12.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/wal
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/wal
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/data/data
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:12.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.974745 17176 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.976342 17176 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:12.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:12.976859 17176 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:13.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.080859 17176 rpc_server.cc:307] RPC server started. Bound to: 127.16.170.61:46189
07:20:13.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.080951 17271 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.170.61:46189 every 8 connection(s)
07:20:13.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.084093 17176 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/data/info.pb
07:20:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.091158 17272 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:20:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.091349 17064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 17176
07:20:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.091874 17064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-1/wal/instance
07:20:13.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.094503 17064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/wal
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/data
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/logs
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/data/info.pb
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:13.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:13.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.60:38441
07:20:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.60
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:13.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json with env {}
07:20:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.109790 17272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } has no permanent_uuid. Determining permanent_uuid...
07:20:13.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.157136 17272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } has no permanent_uuid. Determining permanent_uuid...
07:20:13.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.180217 17272 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } has no permanent_uuid. Determining permanent_uuid...
07:20:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.184178 17207 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.189729 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.239326 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 1
07:20:13.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.245594 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.314091 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 2
07:20:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.318663 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.410127 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 3
07:20:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.415158 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.450899 17274 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.451440 17274 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:13.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.451894 17274 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:13.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.483835 17274 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:13.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.484102 17274 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:13.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.484395 17274 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.484788 17274 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.485325 17274 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.485498 17274 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:39411/jwks.json
07:20:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.520913 17274 master_runner.cc:386] Master server non-default flags:
07:20:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/data
07:20:13.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/wal
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.60:38441
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39411/jwks.json
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:13.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/data/info.pb
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.60
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/logs
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:13.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:13.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:13.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:13.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:13.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.522089 17274 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:13.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.524272 17274 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:13.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.547580 17290 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:13.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.558616 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 4
07:20:13.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.564039 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.572283 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } attempt: 7
07:20:13.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.610332 17173 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } has no permanent_uuid. Determining permanent_uuid...
07:20:13.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.620476 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.678969 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 1
07:20:13.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.684487 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.765983 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 2
07:20:13.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.771678 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.857172 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 3
07:20:13.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.862846 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:13.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:13.863626 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 5
07:20:13.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.869077 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:14.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.003320 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 4
07:20:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:14.009097 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:14.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.292757 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 5
07:20:14.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:14.298468 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:14.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.416611 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 6
07:20:14.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:14.424772 17272 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:14.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.850910 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 6
07:20:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:14.857604 17173 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.16.170.60:38441: Network error: Client connection negotiation failed: client connection to 127.16.170.60:38441: connect: Connection refused (error 111)
07:20:14.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.548329 17292 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:14.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:13.550397 17294 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:14.877832 17293 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:20:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.877880 17274 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:20:14.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.881718 17274 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:14.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.884764 17274 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:14.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.886160 17274 hybrid_clock.cc:648] HybridClock initialized: now 1786000814886112 us; error 66 us; skew 500 ppm
07:20:14.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.887053 17274 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:14.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.894148 17274 webserver.cc:460] Webserver started at http://127.16.170.60:46867/ using document root <none> and password file <none>
07:20:14.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.895135 17274 fs_manager.cc:359] Metadata directory not provided
07:20:14.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.895380 17274 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:14.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.895951 17274 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.901031 17274 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/data/instance:
07:20:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b022aa0372447b3b0c05f0ecfd51fb5"
07:20:14.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:14 on dist-test-slave-04bb"
07:20:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.902346 17274 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/wal/instance:
07:20:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b022aa0372447b3b0c05f0ecfd51fb5"
07:20:14.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:14 on dist-test-slave-04bb"
07:20:14.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.909920 17274 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
07:20:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.915696 17315 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.916887 17274 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
07:20:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.917237 17274 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/wal
07:20:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b022aa0372447b3b0c05f0ecfd51fb5"
07:20:14.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:14 on dist-test-slave-04bb"
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.917604 17274 fs_report.cc:383] FS layout report
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/wal
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/wal
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/data/data
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:14.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:14.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.952376 17274 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:14.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.954023 17274 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:14.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:14.954545 17274 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:15.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.029414 17274 rpc_server.cc:307] RPC server started. Bound to: 127.16.170.60:38441
07:20:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.029542 17377 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.170.60:38441 every 8 connection(s)
07:20:15.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.032243 17274 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/data/info.pb
07:20:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.036471 17064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 17274
07:20:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.036984 17064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/master-2/wal/instance
07:20:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.038147 17378 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:20:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.056128 17378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } has no permanent_uuid. Determining permanent_uuid...
07:20:15.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.083434 17378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } has no permanent_uuid. Determining permanent_uuid...
07:20:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.097565 17378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } has no permanent_uuid. Determining permanent_uuid...
07:20:15.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.122597 17378 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5: Bootstrap starting.
07:20:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.130859 17378 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5: Neither blocks nor log segments found. Creating new log.
07:20:15.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.133044 17378 log.cc:824] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5: Log is configured to *not* fsync() on all Append() calls
07:20:15.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.138332 17378 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5: No bootstrap required, opened a new log
07:20:15.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.160717 17378 raft_consensus.cc:348] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:15.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.161477 17378 raft_consensus.cc:374] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:20:15.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.161806 17378 raft_consensus.cc:729] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3b022aa0372447b3b0c05f0ecfd51fb5, State: Initialized, Role: FOLLOWER
07:20:15.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.162935 17378 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:15.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.165954 17390 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.166728 17390 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: This master's current role is: FOLLOWER
07:20:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.168172 17378 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: configured and running, proceeding with master startup.
07:20:15.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:15.202469 17413 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:15.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:15.202821 17413 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:15.210222 17323 tablet.cc:2369] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:20:15.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.196188 17274 master_runner.cc:186] Error getting master registration for 127.16.170.62:41625: OK, instance_id { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" instance_seqno: 1786000811183757 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:20:15.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.223011 17274 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:20:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.485180 17272 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 7
07:20:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.504550 17272 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: Bootstrap starting.
07:20:15.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.511099 17272 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: Neither blocks nor log segments found. Creating new log.
07:20:15.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.512830 17272 log.cc:824] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: Log is configured to *not* fsync() on all Append() calls
07:20:15.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.517354 17272 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: No bootstrap required, opened a new log
07:20:15.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.536731 17272 raft_consensus.cc:348] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.537935 17272 raft_consensus.cc:374] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:20:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.539000 17272 raft_consensus.cc:729] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6a9f4b3b385344a6b325fddc1c0f608b, State: Initialized, Role: FOLLOWER
07:20:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.540110 17272 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:15.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.543113 17419 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:15.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.543978 17419 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [sys.catalog]: This master's current role is: FOLLOWER
07:20:15.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.544881 17272 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [sys.catalog]: configured and running, proceeding with master startup.
07:20:15.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.578536 17390 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:20:15.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.579105 17390 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:15.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.585440 17240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" is_pre_election: true
07:20:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.586411 17240 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3b022aa0372447b3b0c05f0ecfd51fb5 in term 0.
07:20:15.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.587896 17316 leader_election.cc:302] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3b022aa0372447b3b0c05f0ecfd51fb5, 6a9f4b3b385344a6b325fddc1c0f608b; no voters:
07:20:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.590497 17390 leader_election.cc:288] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42058ffa4c06431aa32f479c7d506370 (127.16.170.62:41625), 6a9f4b3b385344a6b325fddc1c0f608b (127.16.170.61:46189)
07:20:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.591039 17390 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 0 FOLLOWER]: Leader pre-election won for term 1
07:20:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.591346 17390 raft_consensus.cc:482] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:20:15.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.591667 17390 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 0 FOLLOWER]: Advancing to term 1
07:20:15.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.610416 17390 raft_consensus.cc:504] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.617796 17240 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6a9f4b3b385344a6b325fddc1c0f608b"
07:20:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.618348 17240 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [term 0 FOLLOWER]: Advancing to term 1
07:20:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.572682 17176 master_runner.cc:186] Error getting master registration for 127.16.170.62:41625: OK, instance_id { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" instance_seqno: 1786000811183757 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:20:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.620431 17176 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:20:15.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.625517 17240 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3b022aa0372447b3b0c05f0ecfd51fb5 in term 1.
07:20:15.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.643226 17316 leader_election.cc:302] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3b022aa0372447b3b0c05f0ecfd51fb5, 6a9f4b3b385344a6b325fddc1c0f608b; no voters:
07:20:15.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.645591 17390 leader_election.cc:288] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [CANDIDATE]: Term 1 election: Requested vote from peers 42058ffa4c06431aa32f479c7d506370 (127.16.170.62:41625), 6a9f4b3b385344a6b325fddc1c0f608b (127.16.170.61:46189)
07:20:15.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.645665 17437 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 1 FOLLOWER]: Leader election won for term 1
07:20:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.647873 17437 raft_consensus.cc:686] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [term 1 LEADER]: Becoming Leader. State: Replica: 3b022aa0372447b3b0c05f0ecfd51fb5, State: Running, Role: LEADER
07:20:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:15.650147 17433 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:15.650660 17433 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:15.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.651211 17437 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:15.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.669677 17390 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3b022aa0372447b3b0c05f0ecfd51fb5. Latest consensus state: current_term: 1 leader_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:15.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.670249 17390 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: This master's current role is: LEADER
07:20:15.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.672859 17439 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:20:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.681147 17439 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:20:15.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.702803 17240 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [term 1 FOLLOWER]: Refusing update from remote peer 3b022aa0372447b3b0c05f0ecfd51fb5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:20:15.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.704677 17390 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:20:15.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:15.736264 17211 tablet.cc:2369] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:20:15.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.741730 17419 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3b022aa0372447b3b0c05f0ecfd51fb5. Latest consensus state: current_term: 1 leader_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:15.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.742527 17419 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [sys.catalog]: This master's current role is: FOLLOWER
07:20:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.761003 17419 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.761722 17419 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b [sys.catalog]: This master's current role is: FOLLOWER
07:20:15.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.772119 17390 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:15.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.772898 17390 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: This master's current role is: LEADER
07:20:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.775139 17439 catalog_manager.cc:1347] Generated new cluster ID: 06020cab52fe462bb1b21026895da531
07:20:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.775465 17439 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:20:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.869376 17439 catalog_manager.cc:1370] Generated new certificate authority record
07:20:15.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.872085 17439 catalog_manager.cc:1504] Loading token signing keys...
07:20:15.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.891004 17173 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } attempt: 7
07:20:15.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.892288 17439 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5: Generated new TSK 0
07:20:15.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.893543 17439 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:20:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.937377 17173 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370: Bootstrap starting.
07:20:15.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.945674 17173 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370: Neither blocks nor log segments found. Creating new log.
07:20:15.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.948043 17173 log.cc:824] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370: Log is configured to *not* fsync() on all Append() calls
07:20:15.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.961371 17064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/wal
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/data
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/logs
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/data/info.pb
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:15.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.1:0
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.170.1
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.1
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:15.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:20:15.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.962244 17173 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370: No bootstrap required, opened a new log
07:20:15.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.994534 17173 raft_consensus.cc:348] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.995800 17173 raft_consensus.cc:374] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:20:15.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.996351 17173 raft_consensus.cc:729] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42058ffa4c06431aa32f479c7d506370, State: Initialized, Role: FOLLOWER
07:20:15.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:15.997592 17173 consensus_queue.cc:260] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } }
07:20:16.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.013711 17452 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:16.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.014902 17452 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [sys.catalog]: This master's current role is: FOLLOWER
07:20:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.015750 17173 sys_catalog.cc:564] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [sys.catalog]: configured and running, proceeding with master startup.
07:20:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.016376 17132 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [term 0 FOLLOWER]: Advancing to term 1
07:20:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.020895 17135 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42058ffa4c06431aa32f479c7d506370" is_pre_election: true
07:20:16.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.022687 17134 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42058ffa4c06431aa32f479c7d506370"
07:20:16.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.032159 17132 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [term 1 FOLLOWER]: Refusing update from remote peer 3b022aa0372447b3b0c05f0ecfd51fb5: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:20:16.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.038659 17437 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
07:20:16.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.129160 17452 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3b022aa0372447b3b0c05f0ecfd51fb5. Latest consensus state: current_term: 1 leader_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:16.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.129880 17452 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [sys.catalog]: This master's current role is: FOLLOWER
07:20:16.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.130731 17465 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:16.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.131726 17465 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370 [sys.catalog]: This master's current role is: FOLLOWER
07:20:16.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.194134 17476 catalog_manager.cc:1259] Loaded cluster ID: 06020cab52fe462bb1b21026895da531
07:20:16.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.194790 17476 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370: loading cluster ID for follower catalog manager: success
07:20:16.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.204231 17476 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370: acquiring CA information for follower catalog manager: success
07:20:16.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.211105 17476 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 42058ffa4c06431aa32f479c7d506370: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:20:16.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:16.584707 17449 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:16.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:16.585372 17449 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:16.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:16.586180 17449 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:16.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.636773 17437 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42058ffa4c06431aa32f479c7d506370" member_type: VOTER last_known_addr { host: "127.16.170.62" port: 41625 } } peers { permanent_uuid: "6a9f4b3b385344a6b325fddc1c0f608b" member_type: VOTER last_known_addr { host: "127.16.170.61" port: 46189 } } peers { permanent_uuid: "3b022aa0372447b3b0c05f0ecfd51fb5" member_type: VOTER last_known_addr { host: "127.16.170.60" port: 38441 } } }
07:20:16.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.637729 17437 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3b022aa0372447b3b0c05f0ecfd51fb5 [sys.catalog]: This master's current role is: LEADER
07:20:16.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.656426 17433 catalog_manager.cc:1259] Loaded cluster ID: 06020cab52fe462bb1b21026895da531
07:20:16.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.657013 17433 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: loading cluster ID for follower catalog manager: success
07:20:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:16.669085 17449 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:16.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.669695 17433 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: acquiring CA information for follower catalog manager: success
07:20:16.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:16.670465 17449 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.170.1
07:20:16.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.677637 17433 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 6a9f4b3b385344a6b325fddc1c0f608b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:20:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.732019 17449 tablet_server_runner.cc:78] Tablet server non-default flags:
07:20:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/data
07:20:16.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/wal
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.1:0
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/data/info.pb
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.1
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.170.1
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/logs
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:20:16.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:16.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.733672 17449 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:16.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.735752 17449 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:16.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:16.787329 17489 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:16.789531 17449 server_base.cc:1034] running on GCE node
07:20:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:16.789780 17491 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:18.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:16.789218 17497 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:18.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.278224 17449 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:18.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.281284 17449 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:18.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.282742 17449 hybrid_clock.cc:648] HybridClock initialized: now 1786000818282699 us; error 58 us; skew 500 ppm
07:20:18.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.283823 17449 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:18.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.304739 17449 webserver.cc:460] Webserver started at http://127.16.170.1:39919/ using document root <none> and password file <none>
07:20:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.306017 17449 fs_manager.cc:359] Metadata directory not provided
07:20:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.306392 17449 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:18.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.306972 17449 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:18.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.314463 17449 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/data/instance:
07:20:18.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c3283abd89544a58eb8712a60104574"
07:20:18.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:18 on dist-test-slave-04bb"
07:20:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.315934 17449 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/wal/instance:
07:20:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c3283abd89544a58eb8712a60104574"
07:20:18.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:18 on dist-test-slave-04bb"
07:20:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.325888 17449 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
07:20:18.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.333446 17503 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.334703 17449 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:20:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.335072 17449 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/wal
07:20:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7c3283abd89544a58eb8712a60104574"
07:20:18.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:18 on dist-test-slave-04bb"
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.335477 17449 fs_report.cc:383] FS layout report
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/wal
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/wal
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/data/data
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:18.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.402110 17449 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.404107 17449 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.404731 17449 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:18.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.408444 17449 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:20:18.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.414603 17449 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:20:18.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.414870 17449 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:20:18.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.415186 17449 ts_tablet_manager.cc:610] Registered 0 tablets
07:20:18.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.415400 17449 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:20:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.631711 17449 rpc_server.cc:307] RPC server started. Bound to: 127.16.170.1:38537
07:20:18.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.631826 17653 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.170.1:38537 every 8 connection(s)
07:20:18.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.634824 17449 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/data/info.pb
07:20:18.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.646332 17064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 17449
07:20:18.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.646868 17064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-0/wal/instance
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.683488 17064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/wal
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/data
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:18.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/logs
07:20:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/data/info.pb
07:20:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:18.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.2:0
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.170.2
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.2
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:18.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:20:18.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.747092 17657 heartbeater.cc:347] Connected to a master server at 127.16.170.60:38441
07:20:18.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.747623 17657 heartbeater.cc:464] Registering TS with master...
07:20:18.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.749073 17657 heartbeater.cc:511] Master 127.16.170.60:38441 requested a full tablet report, sending...
07:20:18.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.752386 17335 ts_manager.cc:194] Registered new tserver with Master: 7c3283abd89544a58eb8712a60104574 (127.16.170.1:38537)
07:20:18.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.755349 17335 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.170.1:51083
07:20:18.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.757308 17658 heartbeater.cc:347] Connected to a master server at 127.16.170.61:46189
07:20:18.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.757648 17658 heartbeater.cc:464] Registering TS with master...
07:20:18.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.758530 17658 heartbeater.cc:511] Master 127.16.170.61:46189 requested a full tablet report, sending...
07:20:18.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.759994 17664 heartbeater.cc:347] Connected to a master server at 127.16.170.62:41625
07:20:18.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.760332 17664 heartbeater.cc:464] Registering TS with master...
07:20:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.761507 17227 ts_manager.cc:194] Registered new tserver with Master: 7c3283abd89544a58eb8712a60104574 (127.16.170.1:38537)
07:20:18.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.761566 17664 heartbeater.cc:511] Master 127.16.170.62:41625 requested a full tablet report, sending...
07:20:18.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:18.764887 17121 ts_manager.cc:194] Registered new tserver with Master: 7c3283abd89544a58eb8712a60104574 (127.16.170.1:38537)
07:20:19.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:19.096072 17670 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:19.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:19.096745 17670 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:19.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:19.097487 17670 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:19.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:19.136644 17670 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:19.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:19.137611 17670 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.170.2
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:19.175050 17670 tablet_server_runner.cc:78] Tablet server non-default flags:
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/data
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/wal
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.2:0
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/data/info.pb
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.2
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.170.2
07:20:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/logs
07:20:19.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:19.176637 17670 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:19.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:19.178676 17670 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:19.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:19.204133 17677 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:19.767642 17657 heartbeater.cc:503] Master 127.16.170.60:38441 was elected leader, sending a full tablet report...
07:20:21.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:19.210225 17679 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.026813 17670 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.821s user 0.706s sys 1.064s
07:20:21.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.027467 17670 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.823s user 0.707s sys 1.065s
07:20:21.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:20.597812 17676 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 17670
07:20:21.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.029531 17683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.033025 17680 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1827 milliseconds
07:20:21.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.033063 17670 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:20:21.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.034483 17670 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:21.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.037093 17670 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:21.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.038522 17670 hybrid_clock.cc:648] HybridClock initialized: now 1786000821038412 us; error 113 us; skew 500 ppm
07:20:21.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.039572 17670 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:21.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.047694 17670 webserver.cc:460] Webserver started at http://127.16.170.2:41111/ using document root <none> and password file <none>
07:20:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.048991 17670 fs_manager.cc:359] Metadata directory not provided
07:20:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.049271 17670 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:21.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.049786 17670 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.056741 17670 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/data/instance:
07:20:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8590d1ff89cf4e0da8f9e9b05270b798"
07:20:21.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:21 on dist-test-slave-04bb"
07:20:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.058271 17670 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/wal/instance:
07:20:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8590d1ff89cf4e0da8f9e9b05270b798"
07:20:21.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:21 on dist-test-slave-04bb"
07:20:21.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.066004 17670 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.000s sys 0.006s
07:20:21.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.072069 17693 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.073170 17670 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
07:20:21.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.073578 17670 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/wal
07:20:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8590d1ff89cf4e0da8f9e9b05270b798"
07:20:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:21 on dist-test-slave-04bb"
07:20:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.073948 17670 fs_report.cc:383] FS layout report
07:20:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/wal
07:20:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/wal
07:20:21.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/data/data
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:21.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:21.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.113916 17670 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:21.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.115485 17670 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:21.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.115996 17670 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:21.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.118609 17670 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:20:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.122975 17670 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:20:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.123206 17670 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:20:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.123533 17670 ts_tablet_manager.cc:610] Registered 0 tablets
07:20:21.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.123731 17670 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:20:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.279850 17670 rpc_server.cc:307] RPC server started. Bound to: 127.16.170.2:38855
07:20:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.279965 17838 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.170.2:38855 every 8 connection(s)
07:20:21.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.282469 17670 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/data/info.pb
07:20:21.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.285420 17064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 17670
07:20:21.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.286137 17064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-1/wal/instance
07:20:21.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.308355 17064 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/wal
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/data
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/logs
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/data/info.pb
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.3:0
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.170.3
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.3
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:21.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:20:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.335572 17839 heartbeater.cc:347] Connected to a master server at 127.16.170.60:38441
07:20:21.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.336167 17839 heartbeater.cc:464] Registering TS with master...
07:20:21.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.337842 17841 heartbeater.cc:347] Connected to a master server at 127.16.170.62:41625
07:20:21.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.338264 17841 heartbeater.cc:464] Registering TS with master...
07:20:21.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.338338 17839 heartbeater.cc:511] Master 127.16.170.60:38441 requested a full tablet report, sending...
07:20:21.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.339354 17841 heartbeater.cc:511] Master 127.16.170.62:41625 requested a full tablet report, sending...
07:20:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.341745 17335 ts_manager.cc:194] Registered new tserver with Master: 8590d1ff89cf4e0da8f9e9b05270b798 (127.16.170.2:38855)
07:20:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.342674 17121 ts_manager.cc:194] Registered new tserver with Master: 8590d1ff89cf4e0da8f9e9b05270b798 (127.16.170.2:38855)
07:20:21.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.343732 17335 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.170.2:57493
07:20:21.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.345808 17840 heartbeater.cc:347] Connected to a master server at 127.16.170.61:46189
07:20:21.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.346150 17840 heartbeater.cc:464] Registering TS with master...
07:20:21.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.347370 17840 heartbeater.cc:511] Master 127.16.170.61:46189 requested a full tablet report, sending...
07:20:21.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.350039 17227 ts_manager.cc:194] Registered new tserver with Master: 8590d1ff89cf4e0da8f9e9b05270b798 (127.16.170.2:38855)
07:20:21.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.650118 17854 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:21.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.650594 17854 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:21.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.651031 17854 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:21.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.682075 17854 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.682868 17854 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.170.3
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.719794 17854 tablet_server_runner.cc:78] Tablet server non-default flags:
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.170.20:36385
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/data
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/wal
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.170.3:0
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/data/info.pb
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.170.3
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.170.62:41625,127.16.170.61:46189,127.16.170.60:38441
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.170.3
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/logs
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:21.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:21.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.721091 17854 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:21.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:21.722838 17854 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.742440 17868 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.348351 17839 heartbeater.cc:503] Master 127.16.170.60:38441 was elected leader, sending a full tablet report...
07:20:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.742453 17866 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:21.743052 17870 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:22.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:22.942569 17869 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:20:22.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.942715 17854 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:20:22.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.946650 17854 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:22.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.949465 17854 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.950881 17854 hybrid_clock.cc:648] HybridClock initialized: now 1786000822950826 us; error 72 us; skew 500 ppm
07:20:22.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.951717 17854 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.958837 17854 webserver.cc:460] Webserver started at http://127.16.170.3:36295/ using document root <none> and password file <none>
07:20:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.959753 17854 fs_manager.cc:359] Metadata directory not provided
07:20:22.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.960008 17854 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:22.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.960578 17854 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.965588 17854 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/data/instance:
07:20:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bb0339faf484afd923000e878dda478"
07:20:22.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:22 on dist-test-slave-04bb"
07:20:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.966835 17854 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/wal/instance:
07:20:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bb0339faf484afd923000e878dda478"
07:20:22.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:22 on dist-test-slave-04bb"
07:20:22.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.974159 17854 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
07:20:22.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.979823 17882 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.980931 17854 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
07:20:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.981276 17854 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/wal
07:20:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4bb0339faf484afd923000e878dda478"
07:20:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:22 on dist-test-slave-04bb"
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:22.981626 17854 fs_report.cc:383] FS layout report
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/wal
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/wal
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/data/data
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:22.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.028190 17854 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.030309 17854 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:23.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.030875 17854 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.034163 17854 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:20:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.039949 17854 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:20:23.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.040228 17854 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:20:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.040540 17854 ts_tablet_manager.cc:610] Registered 0 tablets
07:20:23.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.040737 17854 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
07:20:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.204228 17854 rpc_server.cc:307] RPC server started. Bound to: 127.16.170.3:44355
07:20:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.204310 18024 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.16.170.3:44355 every 8 connection(s)
07:20:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.207667 17854 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/data/info.pb
07:20:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.216383 17064 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 17854
07:20:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.216887 17064 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster6197685134565032515/ts-2/wal/instance
07:20:23.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.269765 18027 heartbeater.cc:347] Connected to a master server at 127.16.170.62:41625
07:20:23.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.270327 18027 heartbeater.cc:464] Registering TS with master...
07:20:23.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.271389 18025 heartbeater.cc:347] Connected to a master server at 127.16.170.60:38441
07:20:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.271661 18026 heartbeater.cc:347] Connected to a master server at 127.16.170.61:46189
07:20:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.271723 18027 heartbeater.cc:511] Master 127.16.170.62:41625 requested a full tablet report, sending...
07:20:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.271795 18025 heartbeater.cc:464] Registering TS with master...
07:20:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.272107 18026 heartbeater.cc:464] Registering TS with master...
07:20:23.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.272879 18025 heartbeater.cc:511] Master 127.16.170.60:38441 requested a full tablet report, sending...
07:20:23.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.273125 18026 heartbeater.cc:511] Master 127.16.170.61:46189 requested a full tablet report, sending...
07:20:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.275395 17121 ts_manager.cc:194] Registered new tserver with Master: 4bb0339faf484afd923000e878dda478 (127.16.170.3:44355)
07:20:23.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.275605 17335 ts_manager.cc:194] Registered new tserver with Master: 4bb0339faf484afd923000e878dda478 (127.16.170.3:44355)
07:20:23.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.277323 17227 ts_manager.cc:194] Registered new tserver with Master: 4bb0339faf484afd923000e878dda478 (127.16.170.3:44355)
07:20:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.278460 17335 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.16.170.3:54483
07:20:23.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:23.288688 17064 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:20:23.303 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:20:23.304 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:20:23.317 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.170.62"
port: 41625
}
bound_http_address {
host: "127.16.170.62"
port: 37331
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.170.61"
port: 46189
}
bound_http_address {
host: "127.16.170.61"
port: 44465
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.170.60"
port: 38441
}
bound_http_address {
host: "127.16.170.60"
port: 46867
}
}
}
07:20:23.322 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:20:23.325 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.16.170.1"
port: 38537
}
bound_http_address {
host: "127.16.170.1"
port: 39919
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.16.170.2"
port: 38855
}
bound_http_address {
host: "127.16.170.2"
port: 41111
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.16.170.3"
port: 44355
}
bound_http_address {
host: "127.16.170.3"
port: 36295
}
}
}
07:20:23.325 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:20:23.530 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:20:23.654 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNA4_Ab/test-tmp
07:20:23.655 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:20:23.655 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNA4_Ab/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:20:23.658 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:20:24.078 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:24.078 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:20:24.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:24.082090 18040 mini_oidc.cc:152] Starting JWKS server
07:20:24.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:24.089260 18040 webserver.cc:460] Webserver started at http://127.0.0.1:33453/ using document root <none> and password file <none>
07:20:24.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:24.089797 18040 mini_oidc.cc:189] Starting OIDC Discovery server
07:20:24.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:24.091889 18040 webserver.cc:460] Webserver started at http://127.0.0.1:41963/ using document root <none> and password file <none>
07:20:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:20:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/principal' for realm 'KRBTEST.COM',
07:20:24.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:20:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:24 dist-test-slave-04bb krb5kdc[18056](info): setting up network...
07:20:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:20:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:24 dist-test-slave-04bb krb5kdc[18056](info): set up 2 sockets
07:20:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:24 dist-test-slave-04bb krb5kdc[18056](info): commencing operation
07:20:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:20:24.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:24.282548 18025 heartbeater.cc:503] Master 127.16.170.60:38441 was elected leader, sending a full tablet report...
07:20:25.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:25.519456 18040 mini_kdc.cc:121] Time spent starting KDC: real 1.397s user 0.006s sys 0.021s
07:20:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:20:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:20:25.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:20:25.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:20:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:20:25.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:20:25.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:20:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:20:25.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:20:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:20:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/test-user.keytab.
07:20:25.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/test-user.keytab.
07:20:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:25 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000825, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:25.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:20:25.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:20:25Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:20:25.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:20:25Z Disabled control of system clock
07:20:25.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.158.62@KRBTEST.COM; defaulting to no policy
07:20:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:20:25.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.158.62@KRBTEST.COM" created.
07:20:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:20:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.158.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.62.keytab.
07:20:25.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.158.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.62.keytab.
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:25.734107 18040 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/logs
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/info.pb
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:25.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.62:33857
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.62
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.62.keytab
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.62
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:25.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:20:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.046734 18071 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.047428 18071 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:26.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.048182 18071 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.079285 18071 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:20:26.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.079569 18071 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.079996 18071 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.080304 18071 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.080511 18071 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:26.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.080775 18071 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.080920 18071 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:26.126994 18071 master_runner.cc:386] Master server non-default flags:
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.62.keytab
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.62
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.62:33857
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/info.pb
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.62
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/logs
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:26.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:26.128158 18071 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:26.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:26.129205 18071 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:26.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.146966 18080 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:27.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.146977 18081 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:27.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:26.147168 18083 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:26.149343 18071 server_base.cc:1034] running on GCE node
07:20:27.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.441458 18071 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:27.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.444772 18071 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:27.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.446269 18071 hybrid_clock.cc:648] HybridClock initialized: now 1786000827446148 us; error 84 us; skew 500 ppm
07:20:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.447065 18071 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:27.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:27 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000827, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:27.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.453120 18071 init.cc:377] Logged in from keytab as kudu/127.17.158.62@KRBTEST.COM (short username kudu)
07:20:27.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.458592 18071 webserver.cc:460] Webserver started at http://127.17.158.62:44211/ using document root <none> and password file <none>
07:20:27.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.459775 18071 fs_manager.cc:359] Metadata directory not provided
07:20:27.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.459987 18071 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:27.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.460491 18071 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:27.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.465353 18071 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/instance:
07:20:27.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3b490b11b8d487eb8b1be248bfce916"
07:20:27.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:27 on dist-test-slave-04bb"
07:20:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.466372 18071 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal/instance:
07:20:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3b490b11b8d487eb8b1be248bfce916"
07:20:27.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:27 on dist-test-slave-04bb"
07:20:27.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.473876 18071 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
07:20:27.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.479226 18099 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.480176 18071 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
07:20:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.480594 18071 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3b490b11b8d487eb8b1be248bfce916"
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:27 on dist-test-slave-04bb"
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.480940 18071 fs_report.cc:383] FS layout report
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/data
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:27.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:27.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.518344 18071 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:27.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.524652 18071 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.525180 18071 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:27.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.601289 18071 rpc_server.cc:307] RPC server started. Bound to: 127.17.158.62:33857
07:20:27.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.601351 18168 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.158.62:33857 every 8 connection(s)
07:20:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.604354 18071 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/info.pb
07:20:27.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.606600 18040 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 18071
07:20:27.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.607177 18040 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal/instance
07:20:27.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.610953 18169 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:20:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.623143 18169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } has no permanent_uuid. Determining permanent_uuid...
07:20:27.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.158.61@KRBTEST.COM; defaulting to no policy
07:20:27.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:20:27.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.158.61@KRBTEST.COM" created.
07:20:27.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:27 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000827, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.62@KRBTEST.COM for kudu/127.17.158.62@KRBTEST.COM
07:20:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:20:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.158.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.61.keytab.
07:20:27.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.158.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.61.keytab.
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.662599 18040 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/logs
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/info.pb
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.61:35619
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.61
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.61.keytab
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.61
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:27.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:20:27.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.665742 18169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } has no permanent_uuid. Determining permanent_uuid...
07:20:27.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:27.668053 18103 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:27.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:27.672618 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.61:35619: Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.722149 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } attempt: 1
07:20:27.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:27.726035 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.61:35619: Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:27.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.794426 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } attempt: 2
07:20:27.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:27.798584 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.61:35619: Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:27.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:27.890010 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } attempt: 3
07:20:27.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:27.893952 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.61:35619: Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:27.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:27.988304 18176 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:27.989039 18176 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:27.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:27.989753 18176 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.023034 18176 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:20:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.023321 18176 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:28.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.023693 18176 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.024013 18176 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.024313 18176 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.024632 18176 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.024866 18176 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
07:20:28.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:28.037346 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } attempt: 4
07:20:28.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.041718 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.61:35619: Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:28.063963 18176 master_runner.cc:386] Master server non-default flags:
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.61.keytab
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.61
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.61:35619
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/info.pb
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.61
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/logs
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:28.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:28.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:28.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:28.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:28.065526 18176 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:28.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:28.066896 18176 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:28.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.086496 18186 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:28.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:28.341144 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } attempt: 5
07:20:28.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.347805 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.61:35619: Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:28.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:28.895383 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } attempt: 6
07:20:28.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.907743 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.61:35619: Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:29.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.087379 18188 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:29.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:28.087944 18190 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:29.444262 18189 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1356 milliseconds
07:20:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.444373 18176 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:20:29.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.449540 18176 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:29.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.452067 18176 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:29.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.453449 18176 hybrid_clock.cc:648] HybridClock initialized: now 1786000829453416 us; error 33 us; skew 500 ppm
07:20:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.454327 18176 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:29 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000829, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.464329 18176 init.cc:377] Logged in from keytab as kudu/127.17.158.61@KRBTEST.COM (short username kudu)
07:20:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.469837 18176 webserver.cc:460] Webserver started at http://127.17.158.61:43019/ using document root <none> and password file <none>
07:20:29.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.470767 18176 fs_manager.cc:359] Metadata directory not provided
07:20:29.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.471041 18176 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:29.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.471594 18176 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:29.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.476385 18176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/instance:
07:20:29.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d5e4a150e404e8aaee50ac09910beeb"
07:20:29.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:29 on dist-test-slave-04bb"
07:20:29.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.477547 18176 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal/instance:
07:20:29.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d5e4a150e404e8aaee50ac09910beeb"
07:20:29.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:29 on dist-test-slave-04bb"
07:20:29.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.485118 18176 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
07:20:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.490813 18207 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.492033 18176 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
07:20:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.492439 18176 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d5e4a150e404e8aaee50ac09910beeb"
07:20:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:29 on dist-test-slave-04bb"
07:20:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.492897 18176 fs_report.cc:383] FS layout report
07:20:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/data
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:29.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.529114 18176 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.534881 18176 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:29.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.535281 18176 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:29.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.614033 18176 rpc_server.cc:307] RPC server started. Bound to: 127.17.158.61:35619
07:20:29.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.614251 18277 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.158.61:35619 every 8 connection(s)
07:20:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.617961 18176 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/info.pb
07:20:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.623412 18278 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:20:29.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.624277 18040 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 18176
07:20:29.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.624740 18040 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal/instance
07:20:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.639515 18278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } has no permanent_uuid. Determining permanent_uuid...
07:20:29.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.17.158.60@KRBTEST.COM; defaulting to no policy
07:20:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:20:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.17.158.60@KRBTEST.COM" created.
07:20:29.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:29 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000829, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.61@KRBTEST.COM for kudu/127.17.158.62@KRBTEST.COM
07:20:29.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:20:29.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.158.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.60.keytab.
07:20:29.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.17.158.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.60.keytab.
07:20:29.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.677518 18040 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:29.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/logs
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/info.pb
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.60:34689
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.60
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:29.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.60.keytab
07:20:29.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.60
07:20:29.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:20:29.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.680442 18278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } has no permanent_uuid. Determining permanent_uuid...
07:20:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:29 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000829, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.61@KRBTEST.COM for kudu/127.17.158.61@KRBTEST.COM
07:20:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.706476 18278 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } has no permanent_uuid. Determining permanent_uuid...
07:20:29.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:29.708696 18208 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:29.712942 18278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:29.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.762915 18278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 1
07:20:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:29.766672 18278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.835160 18278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 2
07:20:29.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:29.839077 18278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.930541 18278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 3
07:20:29.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:29.934585 18278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.968397 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } attempt: 7
07:20:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:29 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000827, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.62@KRBTEST.COM for kudu/127.17.158.61@KRBTEST.COM
07:20:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:29.993853 18169 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } has no permanent_uuid. Determining permanent_uuid...
07:20:29.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:29.999409 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.006536 18288 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.007066 18288 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.007648 18288 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:30.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.048446 18288 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:20:30.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.048789 18288 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:30.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.049238 18288 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:30.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.049561 18288 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:30.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.049865 18288 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:30.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.050176 18288 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:30.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.050402 18288 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
07:20:30.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.057824 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 1
07:20:30.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.061758 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:30.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.078104 18278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 4
07:20:30.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.082118 18278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.097208 18288 master_runner.cc:386] Master server non-default flags:
07:20:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:30.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.60.keytab
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.60
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.60:34689
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/info.pb
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.60
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/logs
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:30.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.099655 18288 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:30.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.101432 18288 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.121740 18300 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:30.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.143178 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 2
07:20:30.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.148032 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.233438 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 3
07:20:30.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.238320 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:30.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.378841 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 4
07:20:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.381933 18278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 5
07:20:30.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.385123 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.386475 18278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:30.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.668622 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 5
07:20:30.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.673242 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.934063 18278 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 6
07:20:30.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.954556 18278 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:31.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.225695 18169 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } attempt: 6
07:20:31.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:31.232642 18169 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.17.158.60:34689: Network error: Client connection negotiation failed: client connection to 127.17.158.60:34689: connect: Connection refused (error 111)
07:20:31.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.121905 18304 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:31.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:30.121747 18301 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:31.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:30.123329 18288 server_base.cc:1034] running on GCE node
07:20:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.397038 18288 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:31.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.399821 18288 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.401160 18288 hybrid_clock.cc:648] HybridClock initialized: now 1786000831401127 us; error 45 us; skew 500 ppm
07:20:31.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.402192 18288 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:31.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000831, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:31.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.408262 18288 init.cc:377] Logged in from keytab as kudu/127.17.158.60@KRBTEST.COM (short username kudu)
07:20:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.413968 18288 webserver.cc:460] Webserver started at http://127.17.158.60:35497/ using document root <none> and password file <none>
07:20:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.414922 18288 fs_manager.cc:359] Metadata directory not provided
07:20:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.415114 18288 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.415503 18288 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:20:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.420621 18288 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/instance:
07:20:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "241dbb9b73fd44479194bf87494dadfc"
07:20:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:31 on dist-test-slave-04bb"
07:20:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.421741 18288 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal/instance:
07:20:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "241dbb9b73fd44479194bf87494dadfc"
07:20:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:31 on dist-test-slave-04bb"
07:20:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.429087 18288 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
07:20:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.434456 18321 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.435613 18288 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.435895 18288 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "241dbb9b73fd44479194bf87494dadfc"
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:31 on dist-test-slave-04bb"
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.436178 18288 fs_report.cc:383] FS layout report
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/data
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:31.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:31.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.497067 18288 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.503811 18288 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.504279 18288 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.588997 18288 rpc_server.cc:307] RPC server started. Bound to: 127.17.158.60:34689
07:20:31.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.589085 18389 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.158.60:34689 every 8 connection(s)
07:20:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.591604 18288 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/info.pb
07:20:31.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.594084 18040 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 18288
07:20:31.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.594715 18040 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal/instance
07:20:31.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.598392 18390 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:20:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.624087 18390 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } has no permanent_uuid. Determining permanent_uuid...
07:20:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000825, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.158.60@KRBTEST.COM
07:20:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000831, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.60@KRBTEST.COM for kudu/127.17.158.62@KRBTEST.COM
07:20:31.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.673491 18390 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } has no permanent_uuid. Determining permanent_uuid...
07:20:31.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000825, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.158.61@KRBTEST.COM
07:20:31.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000831, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.60@KRBTEST.COM for kudu/127.17.158.61@KRBTEST.COM
07:20:31.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.698551 18390 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } has no permanent_uuid. Determining permanent_uuid...
07:20:31.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000831, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.60@KRBTEST.COM for kudu/127.17.158.60@KRBTEST.COM
07:20:31.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000825, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.158.62@KRBTEST.COM
07:20:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.729894 18040 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:20:31.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.730063 18390 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: Bootstrap starting.
07:20:31.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.735365 18390 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: Neither blocks nor log segments found. Creating new log.
07:20:31.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.737149 18390 log.cc:824] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: Log is configured to *not* fsync() on all Append() calls
07:20:31.738 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:31.740 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:20:31.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.742317 18390 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: No bootstrap required, opened a new log
07:20:31.753 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:20:31.754 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:20:31.758 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.17.158.62"
port: 33857
}
bound_http_address {
host: "127.17.158.62"
port: 44211
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.17.158.61"
port: 35619
}
bound_http_address {
host: "127.17.158.61"
port: 43019
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.17.158.60"
port: 34689
}
bound_http_address {
host: "127.17.158.60"
port: 35497
}
}
}
07:20:31.759 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:20:31.760 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:31.760 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:20:31.762 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:20:31.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.771678 18390 raft_consensus.cc:348] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:31.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.772282 18390 raft_consensus.cc:374] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:20:31.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.772480 18390 raft_consensus.cc:729] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 241dbb9b73fd44479194bf87494dadfc, State: Initialized, Role: FOLLOWER
07:20:31.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.773293 18390 consensus_queue.cc:260] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:31.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.776348 18406 sys_catalog.cc:455] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } } }
07:20:31.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.777269 18406 sys_catalog.cc:458] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [sys.catalog]: This master's current role is: FOLLOWER
07:20:31.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.778077 18390 sys_catalog.cc:564] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [sys.catalog]: configured and running, proceeding with master startup.
07:20:31.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000831, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:31.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:20:31.803 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:31.806 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc
07:20:31.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:31.810199 18421 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:31.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:31.810527 18421 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:31.815 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:20:31.816 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc
07:20:31.818 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:20:31.823 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:20:31.825 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:20:31.825 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:20:31.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.804584 18288 master_runner.cc:186] Error getting master registration for 127.17.158.62:33857: OK, instance_id { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" instance_seqno: 1786000827581403 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:20:31.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.841259 18288 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:20:31.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:31 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000831, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:31.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:20:31.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:31.873 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc
07:20:31.874 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:20:31.875 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.158.62:33857
07:20:31.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:20:31.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.878415 18040 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 18071
07:20:31.912 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:31.913 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.158.61:35619
07:20:31.913 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:20:31.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.914076 18040 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 18176
07:20:31.941 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:31.941 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.17.158.60:34689
07:20:31.942 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:20:31.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.942298 18040 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 18288
07:20:31.975 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:31.975 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.158.62:33857
07:20:31.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:20:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:31.979527 18040 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/logs
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/info.pb
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:31.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.62:33857
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.62
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44211
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.62.keytab
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.62
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:31.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:20:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.304451 18425 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.305172 18425 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:32.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.305902 18425 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:32.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.339308 18425 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:20:32.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.339589 18425 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:32.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.339984 18425 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:32.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.340317 18425 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:32.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.340690 18425 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:32.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.340996 18425 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:32.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.341187 18425 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
07:20:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:32.377847 18425 master_runner.cc:386] Master server non-default flags:
07:20:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.62.keytab
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.62
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.62:33857
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/info.pb
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.62
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44211
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/logs
07:20:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:32.379104 18425 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:32.380138 18425 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.399832 18432 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.399847 18435 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:32.402369 18425 server_base.cc:1034] running on GCE node
07:20:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:32.400624 18431 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:33.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.656970 18425 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.660190 18425 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:33.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.661628 18425 hybrid_clock.cc:648] HybridClock initialized: now 1786000833661604 us; error 44 us; skew 500 ppm
07:20:33.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.662460 18425 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:33.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:33 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000833, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:33.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.668730 18425 init.cc:377] Logged in from keytab as kudu/127.17.158.62@KRBTEST.COM (short username kudu)
07:20:33.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.674252 18425 webserver.cc:460] Webserver started at http://127.17.158.62:44211/ using document root <none> and password file <none>
07:20:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.675132 18425 fs_manager.cc:359] Metadata directory not provided
07:20:33.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.675328 18425 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:33.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.683331 18425 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.002s sys 0.004s
07:20:33.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.687901 18446 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:33.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.689116 18425 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:20:33.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.689416 18425 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:33.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3b490b11b8d487eb8b1be248bfce916"
07:20:33.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:27 on dist-test-slave-04bb"
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.691799 18425 fs_report.cc:383] FS layout report
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/data
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:33.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.729693 18425 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:33.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.735420 18425 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:33.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.735831 18425 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.830972 18425 rpc_server.cc:307] RPC server started. Bound to: 127.17.158.62:33857
07:20:33.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.831108 18509 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.158.62:33857 every 8 connection(s)
07:20:33.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.833876 18425 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/data/info.pb
07:20:33.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.838197 18040 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 18425
07:20:33.838 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:33.839 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.158.61:35619
07:20:33.839 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.840885 18040 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/logs
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/info.pb
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:33.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.61:35619
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.61
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43019
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.61.keytab
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.61
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:33.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:33.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:20:33.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.848694 18510 sys_catalog.cc:263] Verifying existing consensus state
07:20:33.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 07:20:33.850703 18510 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:20:33.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.851197 18425 master.cc:537] Master@127.17.158.62:33857 shutting down...
07:20:33.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:33.922832 18425 master.cc:552] Master@127.17.158.62:33857 shutdown complete.
07:20:33.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:20:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.196476 18513 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.197108 18513 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.197691 18513 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:34.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.230486 18513 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:20:34.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.230712 18513 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:34.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.231118 18513 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:34.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.231464 18513 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.231788 18513 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.232111 18513 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:34.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.232372 18513 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:34.270955 18513 master_runner.cc:386] Master server non-default flags:
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:20:34.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.61.keytab
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.61
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.61:35619
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/info.pb
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.61
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43019
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:34.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/logs
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:34.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:34.272187 18513 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:34.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:34.273478 18513 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:34.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.293195 18525 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.293303 18522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:35.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:34.293206 18523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:35.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:34.294384 18513 server_base.cc:1034] running on GCE node
07:20:35.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.568324 18513 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.571300 18513 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:35.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.572650 18513 hybrid_clock.cc:648] HybridClock initialized: now 1786000835572592 us; error 65 us; skew 500 ppm
07:20:35.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.573535 18513 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:35.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:35 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000835, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:35.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.584602 18513 init.cc:377] Logged in from keytab as kudu/127.17.158.61@KRBTEST.COM (short username kudu)
07:20:35.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.590337 18513 webserver.cc:460] Webserver started at http://127.17.158.61:43019/ using document root <none> and password file <none>
07:20:35.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.591424 18513 fs_manager.cc:359] Metadata directory not provided
07:20:35.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.591702 18513 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.599969 18513 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
07:20:35.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.605089 18539 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:35.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.606202 18513 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.606485 18513 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d5e4a150e404e8aaee50ac09910beeb"
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:29 on dist-test-slave-04bb"
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.609001 18513 fs_report.cc:383] FS layout report
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/data
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:35.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:35.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:35.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.650213 18513 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:35.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.656625 18513 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:35.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.657079 18513 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:35.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.736593 18513 rpc_server.cc:307] RPC server started. Bound to: 127.17.158.61:35619
07:20:35.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.736740 18601 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.158.61:35619 every 8 connection(s)
07:20:35.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.739710 18513 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/data/info.pb
07:20:35.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.743942 18040 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 18513
07:20:35.745 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:35.745 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.17.158.60:34689
07:20:35.746 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.747766 18040 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/logs
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/info.pb
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:35.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.60:34689
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.60
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35497
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.60.keytab
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.60
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:20:35.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.753780 18602 sys_catalog.cc:263] Verifying existing consensus state
07:20:35.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 07:20:35.754760 18602 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:20:35.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.755221 18513 master.cc:537] Master@127.17.158.61:35619 shutting down...
07:20:35.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:35.823086 18513 master.cc:552] Master@127.17.158.61:35619 shutdown complete.
07:20:35.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:20:36.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.089497 18604 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:20:36.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.090050 18604 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:20:36.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.090636 18604 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:20:36.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.127270 18604 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:20:36.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.127537 18604 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:20:36.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.127861 18604 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:20:36.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.128180 18604 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:20:36.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.128376 18604 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:20:36.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.128638 18604 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:20:36.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.128798 18604 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
07:20:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:36.166428 18604 master_runner.cc:386] Master server non-default flags:
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.17.158.20:38039
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/krb5kdc/kudu_127.17.158.60.keytab
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.17.158.60
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.17.158.60:34689
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/info.pb
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.17.158.60
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35497
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:20:36.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/logs
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 06:23:34 UTC on ccycloud.quasar-uqpjlj.root.comops.site
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4569750
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:36.167572 18604 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:20:36.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:36.168663 18604 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:20:36.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.189139 18612 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.190318 18615 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:36.191542 18604 server_base.cc:1034] running on GCE node
07:20:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:36.190768 18611 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:20:37.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.426601 18604 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:20:37.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.429414 18604 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:20:37.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.430832 18604 hybrid_clock.cc:648] HybridClock initialized: now 1786000837430801 us; error 37 us; skew 500 ppm
07:20:37.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.431911 18604 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:20:37.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:37 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000837, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:37.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.438663 18604 init.cc:377] Logged in from keytab as kudu/127.17.158.60@KRBTEST.COM (short username kudu)
07:20:37.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.444206 18604 webserver.cc:460] Webserver started at http://127.17.158.60:35497/ using document root <none> and password file <none>
07:20:37.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.445199 18604 fs_manager.cc:359] Metadata directory not provided
07:20:37.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.445410 18604 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:20:37.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.453220 18604 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
07:20:37.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.457836 18629 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:20:37.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.459048 18604 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.459383 18604 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data,/tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "241dbb9b73fd44479194bf87494dadfc"
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 07:20:31 on dist-test-slave-04bb"
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.462118 18604 fs_report.cc:383] FS layout report
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/wal
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/data
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:20:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:20:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:20:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:20:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:20:37.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:20:37.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.509123 18604 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:20:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.515282 18604 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:20:37.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.515717 18604 kserver.cc:163] Server-wide thread pool size limit: 3276
07:20:37.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.588572 18604 rpc_server.cc:307] RPC server started. Bound to: 127.17.158.60:34689
07:20:37.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.588773 18697 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.17.158.60:34689 every 8 connection(s)
07:20:37.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.591759 18604 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNA4_Ab/test-tmp/mini-kudu-cluster9014230779448652643/master-2/data/info.pb
07:20:37.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.597990 18040 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu as pid 18604
07:20:37.598 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:20:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.604281 18698 sys_catalog.cc:263] Verifying existing consensus state
07:20:37.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.612658 18698 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: Bootstrap starting.
07:20:37.657 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:37.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.659442 18698 log.cc:824] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: Log is configured to *not* fsync() on all Append() calls
07:20:37.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.664592 18698 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:20:37.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.665431 18698 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: Bootstrap complete.
07:20:37.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.691493 18698 raft_consensus.cc:348] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:37.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.694064 18698 raft_consensus.cc:374] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:20:37.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.694324 18698 raft_consensus.cc:729] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 241dbb9b73fd44479194bf87494dadfc, State: Initialized, Role: FOLLOWER
07:20:37.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.695271 18698 consensus_queue.cc:260] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:37.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.698167 18704 sys_catalog.cc:455] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } } }
07:20:37.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.699038 18704 sys_catalog.cc:458] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [sys.catalog]: This master's current role is: FOLLOWER
07:20:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.699868 18698 sys_catalog.cc:564] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [sys.catalog]: configured and running, proceeding with master startup.
07:20:37.720 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:37.721 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.17.158.60:34689(127.17.158.60:34689)] connecting to peer
07:20:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:37.759632 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:37.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:37.760004 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:37.771247 18636 tablet.cc:2369] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:20:37.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.738694 18604 master_runner.cc:186] Error getting master registration for 127.17.158.62:33857: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111),
07:20:37.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:37.782112 18604 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:20:37.782 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:37.782 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:37.783 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:37.785 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:37.786 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:37.788 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:37.789 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.17.158.60:34689(127.17.158.60:34689)] Successfully connected to peer
07:20:37.870 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:20:37.880 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.17.158.60
07:20:38.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:38 dist-test-slave-04bb krb5kdc[18056](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000831, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.17.158.60@KRBTEST.COM
07:20:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:38.172868 18704 raft_consensus.cc:482] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:20:38.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:38.173432 18704 raft_consensus.cc:504] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:38.182837 18630 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111)
07:20:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:38.185087 18704 leader_election.cc:288] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857), 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619)
07:20:38.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:38.209262 18630 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857): Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111)
07:20:38.210 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6d284887, L:/127.0.0.1:55636 - R:/127.17.158.60:34689] using SASL/GSSAPI
07:20:38.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:38.220103 18633 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619): Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:38.220626 18633 leader_election.cc:302] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 241dbb9b73fd44479194bf87494dadfc; no voters: 0d5e4a150e404e8aaee50ac09910beeb, c3b490b11b8d487eb8b1be248bfce916
07:20:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:38.221596 18704 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:20:38.238 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.248 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.259 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.259 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.260 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.265 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.284 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.289 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.289 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.295 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.295 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.296 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.296 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.304 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.305 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.306 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.307 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.307 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.325 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.327 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.327 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.327 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.328 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.338 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.338 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.339 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.339 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.365 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.367 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.374 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.374 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.374 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.375 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.425 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.427 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.427 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.427 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.428 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.432 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.432 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.432 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.434 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.465 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.472 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.472 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.474 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.486 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.487 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.488 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:38.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:38.764565 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:38.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:38.764901 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:38.845 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:38.847 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:38.847 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.848 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:38.854 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.854 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.854 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:38.855 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:39.767935 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:39.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:39.768268 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:39.825 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:39.826 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:39.827 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:39.827 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:39.827 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:39.828 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:39.828 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:39.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:39.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:40.364624 18732 raft_consensus.cc:482] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:20:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:40.364996 18732 raft_consensus.cc:504] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:40.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:40.366607 18732 leader_election.cc:288] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857), 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619)
07:20:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:40.372731 18630 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857): Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111)
07:20:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:40.373216 18633 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619): Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:40.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:40.373587 18633 leader_election.cc:302] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 241dbb9b73fd44479194bf87494dadfc; no voters: 0d5e4a150e404e8aaee50ac09910beeb, c3b490b11b8d487eb8b1be248bfce916
07:20:40.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:40.374264 18732 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:20:40.645 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:40.646 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:40.647 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:40.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:40.648 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:40.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:40.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:40.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:40.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:40.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:40.771397 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:40.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:40.771692 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:41.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:41.774631 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:41.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:41.774892 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:42.205 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:42.206 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:42.207 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:42.207 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:42.208 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:42.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:42.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:42.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:42.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:42.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:42.777743 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:42.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:42.778003 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:42.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:42.785039 18604 master_runner.cc:186] Error getting master registration for 127.17.158.62:33857: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111),
07:20:42.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:42.785405 18604 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:20:42.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:42.797425 18742 raft_consensus.cc:482] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:20:42.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:42.797785 18742 raft_consensus.cc:504] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:42.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:42.799163 18742 leader_election.cc:288] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857), 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619)
07:20:42.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:42.804023 18630 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857): Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111)
07:20:42.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:42.804607 18633 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619): Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:42.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:42.804977 18630 leader_election.cc:302] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 241dbb9b73fd44479194bf87494dadfc; no voters: 0d5e4a150e404e8aaee50ac09910beeb, c3b490b11b8d487eb8b1be248bfce916
07:20:42.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:42.805629 18742 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:20:43.465 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:43.466 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:43.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:43.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:43.468 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:43.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:43.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:43.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:43.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:43.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:43.781359 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:43.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:43.781706 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:44.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:44.785195 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:44.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:44.785524 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:45.585 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:45.587 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:45.587 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:45.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:45.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:45.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:45.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:45.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:45.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:45.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:45.693094 18749 raft_consensus.cc:482] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:20:45.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:45.693563 18749 raft_consensus.cc:504] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:45.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:45.695330 18749 leader_election.cc:288] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857), 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619)
07:20:45.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:45.697476 18630 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:20:45.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:45.700358 18633 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619): Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:45.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:45.700871 18630 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857): Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111)
07:20:45.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:45.701321 18630 leader_election.cc:302] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 241dbb9b73fd44479194bf87494dadfc; no voters: 0d5e4a150e404e8aaee50ac09910beeb, c3b490b11b8d487eb8b1be248bfce916
07:20:45.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:45.702003 18749 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:20:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:45.789146 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:45.789600 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:46.304 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:46.306 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:46.307 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:46.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:46.308 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:46.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:46.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:46.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:46.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:46.793689 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:46.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:46.794085 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:47.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:47 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000847, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:47.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:47.443423 18623 init.cc:295] Successfully reacquired a new kerberos TGT
07:20:47.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:47.788784 18604 master_runner.cc:186] Error getting master registration for 127.17.158.62:33857: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111),
07:20:47.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:47.789196 18604 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:20:47.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:47.797149 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:47.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:47.797411 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:47.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:47.904729 17268 debug-util.cc:398] Leaking SignalData structure 0x7b0800072240 after lost signal to thread 17182
07:20:47.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:47.905606 17268 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3b40 after lost signal to thread 17271
07:20:48.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:48.158155 18757 raft_consensus.cc:482] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:20:48.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:48.158761 18757 raft_consensus.cc:504] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:48.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:48.160977 18757 leader_election.cc:288] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857), 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619)
07:20:48.164 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:48.165 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:48.167300 18633 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619): Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:48.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:48.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:48.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:48.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:48.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:48.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:48.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:48.170462 18630 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857): Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111)
07:20:48.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:48.170787 18630 leader_election.cc:302] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 241dbb9b73fd44479194bf87494dadfc; no voters: 0d5e4a150e404e8aaee50ac09910beeb, c3b490b11b8d487eb8b1be248bfce916
07:20:48.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:48.171460 18757 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:20:48.179 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:48.585 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:48.586 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:48.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:48.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:48.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:48.591 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:48.591 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:48.592 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:48.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:48.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:48.800354 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:48.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:48.800781 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:49.485 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:49.487 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:49.487 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:49.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:49.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:49.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:49.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:49.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:49.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:49.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:49.803824 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:49.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:49.804075 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:49.901270 18766 raft_consensus.cc:482] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:20:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:49.901827 18766 raft_consensus.cc:504] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:20:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:49.903654 18766 leader_election.cc:288] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857), 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619)
07:20:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:49.908612 18630 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857): Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111)
07:20:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:49.909219 18633 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619): Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:20:49.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:49.909551 18633 leader_election.cc:302] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 241dbb9b73fd44479194bf87494dadfc; no voters: 0d5e4a150e404e8aaee50ac09910beeb, c3b490b11b8d487eb8b1be248bfce916
07:20:49.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:49.910219 18766 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:20:50.084 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:50.086 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:50.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:50.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:50.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:50.090 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:50.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:50.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:50.098 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:50.806859 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:50.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:50.807122 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:51.810163 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:51.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:51.810428 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:52.665 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:52.665 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:52.666 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:52.666 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:52.666 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:52.667 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:52.667 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:52.667 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:52.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:52.791790 18604 master_runner.cc:186] Error getting master registration for 127.17.158.62:33857: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111),
07:20:52.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:52.792156 18604 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:20:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:52.813387 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:52.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:52.813647 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:53.816737 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:53.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:53.817003 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:54.820147 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:54.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:54.820469 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:55.823316 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:55.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:55.823621 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:56.604 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:56.605 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:56.606 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:56.606 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:56.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:56.607 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:56.607 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:56.607 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:56.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:56.827638 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:56.827982 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:57.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 07:20:57 dist-test-slave-04bb krb5kdc[18056](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786000857, etypes {rep=17 tkt=17 ses=17}, kudu/127.17.158.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:20:57.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:57.446224 18623 init.cc:295] Successfully reacquired a new kerberos TGT
07:20:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:57.795861 18604 master_runner.cc:186] Error getting master registration for 127.17.158.62:33857: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111),
07:20:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:20:57.796278 18604 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:20:57.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:57.830724 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:57.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:57.830973 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:58.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:58.833716 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:58.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:58.833984 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:20:59.245 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:20:59.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:20:59.247 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:59.247 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:59.247 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:20:59.248 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:59.248 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:59.248 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:20:59.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:20:59.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:59.836834 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:20:59.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:20:59.837173 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:21:00.765 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:21:00.766 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:21:00.766 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:00.766 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:00.767 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:00.768 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:00.768 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:00.768 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:00.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:21:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:00.840029 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:21:00.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:00.840294 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:21:01.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:01.583550 18788 raft_consensus.cc:482] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:21:01.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:01.584131 18788 raft_consensus.cc:504] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c3b490b11b8d487eb8b1be248bfce916" member_type: VOTER last_known_addr { host: "127.17.158.62" port: 33857 } } peers { permanent_uuid: "0d5e4a150e404e8aaee50ac09910beeb" member_type: VOTER last_known_addr { host: "127.17.158.61" port: 35619 } } peers { permanent_uuid: "241dbb9b73fd44479194bf87494dadfc" member_type: VOTER last_known_addr { host: "127.17.158.60" port: 34689 } }
07:21:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:01.586580 18788 leader_election.cc:288] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857), 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619)
07:21:01.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:01.588984 18630 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:21:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:01.592056 18633 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 0d5e4a150e404e8aaee50ac09910beeb (127.17.158.61:35619): Network error: Client connection negotiation failed: client connection to 127.17.158.61:35619: connect: Connection refused (error 111)
07:21:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:01.592614 18630 leader_election.cc:334] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer c3b490b11b8d487eb8b1be248bfce916 (127.17.158.62:33857): Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111)
07:21:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:01.593035 18630 leader_election.cc:302] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 241dbb9b73fd44479194bf87494dadfc; no voters: 0d5e4a150e404e8aaee50ac09910beeb, c3b490b11b8d487eb8b1be248bfce916
07:21:01.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:01.593977 18788 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:21:01.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:01.843501 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:21:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:01.843783 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:21:01.864 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:21:01.865 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:21:01.866 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:01.866 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:01.866 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:01.866 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:01.867 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:01.867 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:01.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:21:02.185 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:21:02.186 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:21:02.187 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:02.187 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:02.187 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:02.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:02.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:02.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:02.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:21:02.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:02.799111 18604 master_runner.cc:186] Error getting master registration for 127.17.158.62:33857: Network error: Client connection negotiation failed: client connection to 127.17.158.62:33857: connect: Connection refused (error 111),
07:21:02.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:02.799445 18604 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:21:02.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:02.847014 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:21:02.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:02.847368 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:21:03.404 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:21:03.405 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:21:03.406 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:03.406 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:03.406 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:03.407 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:03.407 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:03.407 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:03.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:21:03.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:03.850503 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:21:03.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:03.850770 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:21:04.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:04.854082 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:21:04.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:04.854369 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:21:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:05.857910 18717 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 241dbb9b73fd44479194bf87494dadfc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:21:05.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 07:21:05.858191 18717 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:21:05.985 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.62:33857(127.17.158.62:33857)] connecting to peer
07:21:05.985 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.17.158.61:35619(127.17.158.61:35619)] connecting to peer
07:21:05.986 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:05.986 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.17.158.62:33857(127.17.158.62:33857)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:05.986 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.62:33857: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857
07:21:05.987 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:05.987 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.61:35619(127.17.158.61:35619)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:05.987 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.17.158.61:35619: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
07:21:05.989 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689; will retry
07:21:05.993 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=28386), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [175ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [177ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [624ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [650ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [655ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [656ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [679ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [685ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [686ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [687ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [698ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [700ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [702ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [703ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [718ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [727ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [731ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [760ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [764ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [765ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [766ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [819ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [823ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [825ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [825ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [864ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [875ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [878ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [880ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [882ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [883ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1235ms] refreshing cache from master, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [1244ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1246ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2215ms] refreshing cache from master, [2215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [2218ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [2222ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3035ms] refreshing cache from master, [3035ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [3037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [3038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [3039ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [3041ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3042ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [3042ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [4595ms] refreshing cache from master, [4595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [4599ms] too many traces: truncated at 100 traces, deferred=Deferred@127203504(state=PENDING, result=null, callback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926), errback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926)))
07:21:05.998 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.158.61:35619(127.17.158.61:35619)] disconnecting while in state TERMINATED
07:21:06.000 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.158.62:33857(127.17.158.62:33857)] disconnecting while in state TERMINATED
07:21:06.000 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.17.158.60:34689(127.17.158.60:34689)] disconnecting while in state READY
07:21:06.005 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:21:06.007 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.17.158.60:34689(127.17.158.60:34689)] handling channelInactive
07:21:06.007 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.17.158.60:34689(127.17.158.60:34689)] cleaning up while in state READY due to: connection closed
07:21:06.013 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:21:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:06.022187 18040 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 18604
07:21:07.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:21:07Z chronyd exiting
07:21:08.256 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:21:08.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:08.258172 17064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 17449
07:21:08.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:08.291919 17064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 17670
07:21:08.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:08.330283 17064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 17854
07:21:08.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:08.364917 17064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 17081
07:21:08.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:08.402526 17064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 17176
07:21:08.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 07:21:08.446833 17064 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNA4_Ab/build/tsan/bin/kudu with pid 17274
07:21:09.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T07:21:09Z chronyd exiting
07:21:10.664 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=28386), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [175ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [177ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [624ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [650ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [655ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [656ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [679ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [685ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [686ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [687ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [698ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [700ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [702ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [703ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [718ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [727ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [731ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [760ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [764ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [765ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [766ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [819ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [823ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [825ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [825ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [864ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [875ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [878ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [880ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [882ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [883ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1235ms] refreshing cache from master, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [1244ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1246ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2215ms] refreshing cache from master, [2215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [2218ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [2222ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3035ms] refreshing cache from master, [3035ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [3037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [3038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [3039ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [3041ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3042ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [3042ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [4595ms] refreshing cache from master, [4595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [4599ms] too many traces: truncated at 100 traces, deferred=Deferred@127203504(state=PENDING, result=null, callback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926), errback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.158.61:35619
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=28386), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [175ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [177ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [624ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [650ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [655ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [656ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [679ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [685ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [686ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [687ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [698ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [700ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [702ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [703ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [718ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [727ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [731ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [760ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [764ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [765ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [766ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [819ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [823ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [825ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [825ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [864ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [875ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [878ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [880ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [882ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [883ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1235ms] refreshing cache from master, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [1244ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1246ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2215ms] refreshing cache from master, [2215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [2218ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [2222ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3035ms] refreshing cache from master, [3035ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [3037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [3038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [3039ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [3041ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3042ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [3042ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [4595ms] refreshing cache from master, [4595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [4599ms] too many traces: truncated at 100 traces, deferred=Deferred@127203504(state=PENDING, result=null, callback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926), errback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.158.61:35619
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
07:21:11.804 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
07:21:11.804 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=28386), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [175ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [177ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [624ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [650ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [655ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [656ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [679ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [685ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [686ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [687ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [698ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [700ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [702ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [703ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [718ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [727ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [731ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [760ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [764ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [765ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [766ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [819ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [823ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [825ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [825ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [864ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [875ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [878ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [880ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [882ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [883ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1235ms] refreshing cache from master, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [1244ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1246ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2215ms] refreshing cache from master, [2215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [2218ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [2222ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3035ms] refreshing cache from master, [3035ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [3037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [3038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [3039ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [3041ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3042ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [3042ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [4595ms] refreshing cache from master, [4595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [4599ms] too many traces: truncated at 100 traces, deferred=Deferred@127203504(state=PENDING, result=null, callback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926), errback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.158.61:35619
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=28, TimeoutTracker(timeout=30000, elapsed=28386), Traces: [0ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [110ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [175ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [177ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [624ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [629ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [635ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [640ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [650ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [655ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [656ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [675ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [679ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [680ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [685ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [686ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [687ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [695ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [698ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [700ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [702ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [703ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [718ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [720ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [727ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [730ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [731ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [755ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [759ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [760ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [764ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [765ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [766ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [815ms] refreshing cache from master, [815ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [819ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [823ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [825ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [825ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [855ms] refreshing cache from master, [855ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [857ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [861ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [864ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [875ms] refreshing cache from master, [875ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [878ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [880ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [882ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [883ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1235ms] refreshing cache from master, [1236ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [1238ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [1239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [1239ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [1244ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [1245ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [1246ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2215ms] refreshing cache from master, [2215ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [2216ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [2217ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [2218ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [2219ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [2221ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [2222ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3035ms] refreshing cache from master, [3035ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [3037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [3038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [3039ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.62:33857: Network error: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857, [3041ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.61:35619: Network error: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [3042ms] Sub RPC ConnectToMaster: received response from server master-127.17.158.60:34689: OK, [3042ms] delaying RPC due to: Service unavailable: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619, [4595ms] refreshing cache from master, [4595ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.62:33857, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.61:35619, [4597ms] Sub RPC ConnectToMaster: sending RPC to server master-127.17.158.60:34689, [4599ms] too many traces: truncated at 100 traces, deferred=Deferred@127203504(state=PENDING, result=null, callback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926), errback=(continuation of Deferred@1620646695 after retry RPC after error@917373331) -> (continuation of Deferred@709235367 after retry RPC after error@179781159) -> (continuation of Deferred@585059445 after retry RPC after error@1690002212) -> (continuation of Deferred@2121438782 after retry RPC after error@1292360145) -> (continuation of Deferred@367233248 after retry RPC after error@721316667) -> (continuation of Deferred@1087888188 after retry RPC after error@862800806) -> (continuation of Deferred@985686435 after retry RPC after error@1665036915) -> (continuation of Deferred@500058899 after retry RPC after error@1151800183) -> (continuation of Deferred@2102640441 after retry RPC after error@1835984650) -> (continuation of Deferred@35106663 after retry RPC after error@2128029945) -> (continuation of Deferred@505281133 after retry RPC after error@72423443) -> (continuation of Deferred@990045033 after retry RPC after error@2085983248) -> (continuation of Deferred@225105765 after retry RPC after error@24627035) -> (continuation of Deferred@759326135 after retry RPC after error@2112327963) -> (continuation of Deferred@644718653 after retry RPC after error@421521076) -> (continuation of Deferred@667830269 after retry RPC after error@1717593430) -> (continuation of Deferred@768040431 after retry RPC after error@1267414181) -> (continuation of Deferred@506422150 after retry RPC after error@2072051648) -> (continuation of Deferred@1824517498 after retry RPC after error@2122564180) -> (continuation of Deferred@466992465 after retry RPC after error@1733766932) -> (continuation of Deferred@1955646137 after retry RPC after error@350304029) -> (continuation of Deferred@585641957 after retry RPC after error@2082052899) -> (continuation of Deferred@237916955 after retry RPC after error@334152732) -> (continuation of Deferred@1889425027 after retry RPC after error@1089342256) -> (continuation of Deferred@1017396925 after retry RPC after error@2032912145) -> (continuation of Deferred@2063842553 after retry RPC after error@1266982690) -> (continuation of Deferred@1447559481 after retry RPC after error@1912362926)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.17.158.62:33857,127.17.158.61:35619,127.17.158.60:34689) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.62:33857(127.17.158.62:33857): Connection refused: /127.17.158.62:33857,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.17.158.61:35619(127.17.158.61:35619): Connection refused: /127.17.158.61:35619
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.17.158.61:35619
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)