11:49:00.831 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
11:49:00.845 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0xMXX5/test-tmp
11:49:00.847 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:49:00.851 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0xMXX5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:49:01.109 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
11:49:01.293 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:49:01.295 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:49:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.298780 796 mini_oidc.cc:152] Starting JWKS server
11:49:01.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.302902 796 webserver.cc:460] Webserver started at http://127.0.0.1:46717/ using document root <none> and password file <none>
11:49:01.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.303532 796 mini_oidc.cc:189] Starting OIDC Discovery server
11:49:01.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.305917 796 webserver.cc:460] Webserver started at http://127.0.0.1:37639/ using document root <none> and password file <none>
11:49:01.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T11:49:01Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:49:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T11:49:01Z Disabled control of system clock
11:49:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.361786 796 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:01.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/wal
11:49:01.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/data
11:49:01.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:01.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:01.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/logs
11:49:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/data/info.pb
11:49:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.62:34729
11:49:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.62
11:49:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:01.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:01.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46717/jwks.json with env {}
11:49:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.679159 808 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.679724 808 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:01.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.680354 808 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:01.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.712638 808 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:01.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.712966 808 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:01.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.713198 808 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:01.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.713402 808 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:01.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.713793 808 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:01.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.714067 808 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46717/jwks.json
11:49:01.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.750744 808 master_runner.cc:386] Master server non-default flags:
11:49:01.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/data
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/wal
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.62:34729
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:01.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46717/jwks.json
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/data/info.pb
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.62
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:01.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/logs
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.752142 808 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:01.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.754418 808 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:01.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.776098 817 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:03.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.776176 815 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:03.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:01.778090 814 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:03.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:01.777840 808 server_base.cc:1034] running on GCE node
11:49:03.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.014227 808 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:03.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.017544 808 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:03.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.019075 808 hybrid_clock.cc:648] HybridClock initialized: now 1781869743019027 us; error 63 us; skew 500 ppm
11:49:03.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.020062 808 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:03.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.028031 808 webserver.cc:460] Webserver started at http://127.0.199.62:38929/ using document root <none> and password file <none>
11:49:03.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.029172 808 fs_manager.cc:359] Metadata directory not provided
11:49:03.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.029377 808 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.029778 808 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:03.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.035221 808 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/data/instance:
11:49:03.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f03826fba74f27b10c47d56ba2312f"
11:49:03.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:03 on dist-test-slave-kqf6"
11:49:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.036727 808 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/wal/instance:
11:49:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f03826fba74f27b10c47d56ba2312f"
11:49:03.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:03 on dist-test-slave-kqf6"
11:49:03.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.045339 808 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.003s sys 0.005s
11:49:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.051429 824 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:03.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.052937 808 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
11:49:03.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.053349 808 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/wal
11:49:03.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4f03826fba74f27b10c47d56ba2312f"
11:49:03.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:03 on dist-test-slave-kqf6"
11:49:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.053853 808 fs_report.cc:383] FS layout report
11:49:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/wal
11:49:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/wal
11:49:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/data/data
11:49:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:03.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:03.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.100029 808 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:03.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.101763 808 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:03.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.102365 808 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:03.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.185684 808 rpc_server.cc:307] RPC server started. Bound to: 127.0.199.62:34729
11:49:03.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.185814 875 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.199.62:34729 every 8 connection(s)
11:49:03.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.188655 808 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/data/info.pb
11:49:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.194677 796 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 808
11:49:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.195291 796 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-0/wal/instance
11:49:03.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.195443 876 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:49:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.201112 796 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/wal
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/data
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/logs
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/data/info.pb
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:03.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.61:37391
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.61
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:03.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46717/jwks.json with env {}
11:49:03.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.213027 876 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } has no permanent_uuid. Determining permanent_uuid...
11:49:03.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.249236 876 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } has no permanent_uuid. Determining permanent_uuid...
11:49:03.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.251647 828 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.199.61:37391: connect: Connection refused (error 111)
11:49:03.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.256742 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.61:37391: Network error: Client connection negotiation failed: client connection to 127.0.199.61:37391: connect: Connection refused (error 111)
11:49:03.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.306380 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } attempt: 1
11:49:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.310602 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.61:37391: Network error: Client connection negotiation failed: client connection to 127.0.199.61:37391: connect: Connection refused (error 111)
11:49:03.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.379101 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } attempt: 2
11:49:03.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.383121 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.61:37391: Network error: Client connection negotiation failed: client connection to 127.0.199.61:37391: connect: Connection refused (error 111)
11:49:03.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.474637 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } attempt: 3
11:49:03.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.478911 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.61:37391: Network error: Client connection negotiation failed: client connection to 127.0.199.61:37391: connect: Connection refused (error 111)
11:49:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.529071 878 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:03.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.529646 878 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:03.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.530229 878 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.562121 878 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.562407 878 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.562602 878 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:03.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.562783 878 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.562956 878 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.563198 878 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46717/jwks.json
11:49:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.599025 878 master_runner.cc:386] Master server non-default flags:
11:49:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/data
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/wal
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.61:37391
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46717/jwks.json
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/data/info.pb
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.61
11:49:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/logs
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.600236 878 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.602103 878 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:03.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.622521 887 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:03.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.622334 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } attempt: 4
11:49:03.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.627326 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.61:37391: Network error: Client connection negotiation failed: client connection to 127.0.199.61:37391: connect: Connection refused (error 111)
11:49:03.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.926786 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } attempt: 5
11:49:03.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.931489 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.61:37391: Network error: Client connection negotiation failed: client connection to 127.0.199.61:37391: connect: Connection refused (error 111)
11:49:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.478896 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } attempt: 6
11:49:04.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:04.487404 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.61:37391: Network error: Client connection negotiation failed: client connection to 127.0.199.61:37391: connect: Connection refused (error 111)
11:49:04.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.623705 888 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:03.625690 878 server_base.cc:1034] running on GCE node
11:49:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:03.624185 890 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:04.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.872989 878 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:04.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.876706 878 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:04.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.878196 878 hybrid_clock.cc:648] HybridClock initialized: now 1781869744878119 us; error 98 us; skew 500 ppm
11:49:04.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.879511 878 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:04.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.887869 878 webserver.cc:460] Webserver started at http://127.0.199.61:40615/ using document root <none> and password file <none>
11:49:04.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.889374 878 fs_manager.cc:359] Metadata directory not provided
11:49:04.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.889665 878 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:04.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.890257 878 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.896411 878 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/data/instance:
11:49:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2016a02f5314e1d87b335b60d0d05a8"
11:49:04.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:04 on dist-test-slave-kqf6"
11:49:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.897745 878 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/wal/instance:
11:49:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2016a02f5314e1d87b335b60d0d05a8"
11:49:04.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:04 on dist-test-slave-kqf6"
11:49:04.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.906265 878 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.000s sys 0.008s
11:49:04.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.912730 899 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.914063 878 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
11:49:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.914389 878 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/wal
11:49:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2016a02f5314e1d87b335b60d0d05a8"
11:49:04.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:04 on dist-test-slave-kqf6"
11:49:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.914708 878 fs_report.cc:383] FS layout report
11:49:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/wal
11:49:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/wal
11:49:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/data/data
11:49:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:04.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:04.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.952512 878 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:04.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.953989 878 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:04.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:04.954427 878 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.037608 878 rpc_server.cc:307] RPC server started. Bound to: 127.0.199.61:37391
11:49:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.037696 950 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.199.61:37391 every 8 connection(s)
11:49:05.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.040252 878 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/data/info.pb
11:49:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.044821 796 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 878
11:49:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.045802 796 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-1/wal/instance
11:49:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.048099 951 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:49:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.049238 796 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:05.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/wal
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/data
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/logs
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/data/info.pb
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.60:41761
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.60
11:49:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46717/jwks.json with env {}
11:49:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.064261 951 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } has no permanent_uuid. Determining permanent_uuid...
11:49:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.092274 951 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } has no permanent_uuid. Determining permanent_uuid...
11:49:05.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.112713 951 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } has no permanent_uuid. Determining permanent_uuid...
11:49:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.115520 902 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.120121 951 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.169703 951 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 1
11:49:05.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.173731 951 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.242215 951 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 2
11:49:05.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.246513 951 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.337934 951 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 3
11:49:05.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.342545 951 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.407208 953 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:05.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.407840 953 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:05.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.408308 953 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.440157 953 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.440554 953 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:05.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.440757 953 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:05.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.440949 953 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:05.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.441200 953 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:05.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.441354 953 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46717/jwks.json
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.476462 953 master_runner.cc:386] Master server non-default flags:
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/data
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/wal
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.60:41761
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:05.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46717/jwks.json
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/data/info.pb
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.60
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/logs
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.477882 953 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:05.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.479723 953 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:05.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.486037 951 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 4
11:49:05.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.492224 951 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.502856 966 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.547875 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } attempt: 7
11:49:05.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.565276 876 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } has no permanent_uuid. Determining permanent_uuid...
11:49:05.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.575736 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.634225 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 1
11:49:05.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.639539 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.721000 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 2
11:49:05.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.724797 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.791924 951 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 5
11:49:05.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.797490 951 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.810503 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 3
11:49:05.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.814786 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:05.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.955479 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 4
11:49:05.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.960827 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:06.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.244321 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 5
11:49:06.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:06.249696 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.344960 951 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 6
11:49:06.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:06.351855 951 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:06.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.503316 963 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:06.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:05.504720 953 server_base.cc:1034] running on GCE node
11:49:06.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:05.504638 964 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:06.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.757824 953 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:06.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.761304 953 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:06.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.762825 953 hybrid_clock.cc:648] HybridClock initialized: now 1781869746762727 us; error 86 us; skew 500 ppm
11:49:06.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.763656 953 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:06.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.771409 953 webserver.cc:460] Webserver started at http://127.0.199.60:43199/ using document root <none> and password file <none>
11:49:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.772353 953 fs_manager.cc:359] Metadata directory not provided
11:49:06.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.772629 953 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.773067 953 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:06.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.778280 953 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/data/instance:
11:49:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d822ebbc241f4649bf7b949f02457dda"
11:49:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:06 on dist-test-slave-kqf6"
11:49:06.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.779630 953 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/wal/instance:
11:49:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d822ebbc241f4649bf7b949f02457dda"
11:49:06.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:06 on dist-test-slave-kqf6"
11:49:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.790287 953 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.005s sys 0.003s
11:49:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.797955 977 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:06.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.799314 953 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
11:49:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.799618 953 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/wal
11:49:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d822ebbc241f4649bf7b949f02457dda"
11:49:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:06 on dist-test-slave-kqf6"
11:49:06.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.800066 953 fs_report.cc:383] FS layout report
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/wal
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/wal
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/data/data
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:06.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:06.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.802137 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 6
11:49:06.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:06.809232 876 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.199.60:41761: Network error: Client connection negotiation failed: client connection to 127.0.199.60:41761: connect: Connection refused (error 111)
11:49:06.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.837083 953 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:06.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.838668 953 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:06.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.839090 953 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:06.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.921149 953 rpc_server.cc:307] RPC server started. Bound to: 127.0.199.60:41761
11:49:06.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.921249 1030 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.199.60:41761 every 8 connection(s)
11:49:06.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.924227 953 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/data/info.pb
11:49:06.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.931571 1031 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:49:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.932844 796 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 953
11:49:06.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.933275 796 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/master-2/wal/instance
11:49:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.948956 1031 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } has no permanent_uuid. Determining permanent_uuid...
11:49:06.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.980569 1031 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } has no permanent_uuid. Determining permanent_uuid...
11:49:06.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:06.993278 1031 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } has no permanent_uuid. Determining permanent_uuid...
11:49:07.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.013998 1031 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda: Bootstrap starting.
11:49:07.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.024056 1031 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda: Neither blocks nor log segments found. Creating new log.
11:49:07.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.026499 1031 log.cc:824] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda: Log is configured to *not* fsync() on all Append() calls
11:49:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.032197 1031 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda: No bootstrap required, opened a new log
11:49:07.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.067857 1031 raft_consensus.cc:348] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.068758 1031 raft_consensus.cc:374] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:49:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.069031 1031 raft_consensus.cc:729] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d822ebbc241f4649bf7b949f02457dda, State: Initialized, Role: FOLLOWER
11:49:07.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.069897 1031 consensus_queue.cc:260] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:07.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.075448 1031 sys_catalog.cc:564] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: configured and running, proceeding with master startup.
11:49:07.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.074980 1040 sys_catalog.cc:455] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:07.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.076586 1040 sys_catalog.cc:458] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: This master's current role is: FOLLOWER
11:49:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:07.094851 985 tablet.cc:2369] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:49:07.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:07.104139 1051 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:07.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:07.104547 1051 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:07.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.099062 953 master_runner.cc:186] Error getting master registration for 127.0.199.62:34729: OK, instance_id { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" instance_seqno: 1781869743161957 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:49:07.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.136988 953 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.333503 1040 raft_consensus.cc:482] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.334067 1040 raft_consensus.cc:504] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:07.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.336421 1040 leader_election.cc:288] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b4f03826fba74f27b10c47d56ba2312f (127.0.199.62:34729), d2016a02f5314e1d87b335b60d0d05a8 (127.0.199.61:37391)
11:49:07.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.412501 951 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 7
11:49:07.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.448697 951 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8: Bootstrap starting.
11:49:07.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.456560 951 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8: Neither blocks nor log segments found. Creating new log.
11:49:07.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.459364 951 log.cc:824] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8: Log is configured to *not* fsync() on all Append() calls
11:49:07.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.464901 951 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8: No bootstrap required, opened a new log
11:49:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.497812 951 raft_consensus.cc:348] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.498724 951 raft_consensus.cc:374] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:49:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.499015 951 raft_consensus.cc:729] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2016a02f5314e1d87b335b60d0d05a8, State: Initialized, Role: FOLLOWER
11:49:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.499943 951 consensus_queue.cc:260] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:07.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.503360 1055 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.504405 1055 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [sys.catalog]: This master's current role is: FOLLOWER
11:49:07.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.505357 951 sys_catalog.cc:564] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [sys.catalog]: configured and running, proceeding with master startup.
11:49:07.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.505901 926 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d822ebbc241f4649bf7b949f02457dda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d2016a02f5314e1d87b335b60d0d05a8" is_pre_election: true
11:49:07.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.507332 926 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d822ebbc241f4649bf7b949f02457dda in term 0.
11:49:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.509122 983 leader_election.cc:302] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d2016a02f5314e1d87b335b60d0d05a8, d822ebbc241f4649bf7b949f02457dda; no voters:
11:49:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.510066 1040 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 0 FOLLOWER]: Leader pre-election won for term 1
11:49:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.510461 1040 raft_consensus.cc:482] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
11:49:07.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.510835 1040 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 0 FOLLOWER]: Advancing to term 1
11:49:07.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.523151 1040 raft_consensus.cc:504] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:07.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.530323 926 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d822ebbc241f4649bf7b949f02457dda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d2016a02f5314e1d87b335b60d0d05a8"
11:49:07.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.531028 926 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [term 0 FOLLOWER]: Advancing to term 1
11:49:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.541847 926 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d822ebbc241f4649bf7b949f02457dda in term 1.
11:49:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.542922 1040 leader_election.cc:288] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [CANDIDATE]: Term 1 election: Requested vote from peers b4f03826fba74f27b10c47d56ba2312f (127.0.199.62:34729), d2016a02f5314e1d87b335b60d0d05a8 (127.0.199.61:37391)
11:49:07.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.544832 983 leader_election.cc:302] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d2016a02f5314e1d87b335b60d0d05a8, d822ebbc241f4649bf7b949f02457dda; no voters:
11:49:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.545644 1040 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 1 FOLLOWER]: Leader election won for term 1
11:49:07.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.548779 1040 raft_consensus.cc:686] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [term 1 LEADER]: Becoming Leader. State: Replica: d822ebbc241f4649bf7b949f02457dda, State: Running, Role: LEADER
11:49:07.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.550068 1040 consensus_queue.cc:237] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:07.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.576135 1064 sys_catalog.cc:455] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: SysCatalogTable state changed. Reason: New leader d822ebbc241f4649bf7b949f02457dda. Latest consensus state: current_term: 1 leader_uuid: "d822ebbc241f4649bf7b949f02457dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:07.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.577642 1064 sys_catalog.cc:458] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: This master's current role is: LEADER
11:49:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:07.605044 1069 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:07.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:07.605446 1069 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:07.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.605414 1067 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
11:49:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.619433 1067 catalog_manager.cc:1484] Initializing Kudu cluster ID...
11:49:07.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.653667 926 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [term 1 FOLLOWER]: Refusing update from remote peer d822ebbc241f4649bf7b949f02457dda: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:49:07.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.655753 1064 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
11:49:07.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.588274 878 master_runner.cc:186] Error getting master registration for 127.0.199.62:34729: OK, instance_id { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" instance_seqno: 1781869743161957 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:49:07.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.671232 878 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.700733 1071 mvcc.cc:205] Tried to move back new op lower bound from 7298538486344044544 to 7298538486015508480. Current Snapshot: MvccSnapshot[applied={T|T < 7298538486344044544}]
11:49:07.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.707077 1055 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d822ebbc241f4649bf7b949f02457dda. Latest consensus state: current_term: 1 leader_uuid: "d822ebbc241f4649bf7b949f02457dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:07.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.708230 1055 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [sys.catalog]: This master's current role is: FOLLOWER
11:49:07.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.714854 1040 sys_catalog.cc:455] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d822ebbc241f4649bf7b949f02457dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:07.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.715741 1040 sys_catalog.cc:458] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: This master's current role is: LEADER
11:49:07.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.726181 1067 catalog_manager.cc:1347] Generated new cluster ID: 356c71ce9da14ec4b306ddf6971f6e77
11:49:07.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.726966 1067 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
11:49:07.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.744741 1055 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d822ebbc241f4649bf7b949f02457dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:07.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.747862 1055 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8 [sys.catalog]: This master's current role is: FOLLOWER
11:49:07.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.772115 1067 catalog_manager.cc:1370] Generated new certificate authority record
11:49:07.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.779585 1067 catalog_manager.cc:1504] Loading token signing keys...
11:49:07.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.813534 1067 catalog_manager.cc:5868] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda: Generated new TSK 0
11:49:07.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.817603 1067 catalog_manager.cc:1514] Initializing in-progress tserver states...
11:49:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.842653 876 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } attempt: 7
11:49:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.843385 796 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/wal
11:49:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/data
11:49:07.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/logs
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/data/info.pb
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.1:0
11:49:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.199.1
11:49:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.1
11:49:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:49:07.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.896523 876 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f: Bootstrap starting.
11:49:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.904819 876 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f: Neither blocks nor log segments found. Creating new log.
11:49:07.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.907342 876 log.cc:824] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f: Log is configured to *not* fsync() on all Append() calls
11:49:07.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:07.945432 876 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f: No bootstrap required, opened a new log
11:49:08.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.024482 876 raft_consensus.cc:348] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:08.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.026791 876 raft_consensus.cc:374] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:49:08.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.027897 876 raft_consensus.cc:729] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b4f03826fba74f27b10c47d56ba2312f, State: Initialized, Role: FOLLOWER
11:49:08.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.031121 876 consensus_queue.cc:260] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } }
11:49:08.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.058192 1080 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.059374 1080 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [sys.catalog]: This master's current role is: FOLLOWER
11:49:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.065351 851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d822ebbc241f4649bf7b949f02457dda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b4f03826fba74f27b10c47d56ba2312f" is_pre_election: true
11:49:08.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.065394 850 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d822ebbc241f4649bf7b949f02457dda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b4f03826fba74f27b10c47d56ba2312f"
11:49:08.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.076287 849 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [term 0 FOLLOWER]: Advancing to term 1
11:49:08.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.083988 876 sys_catalog.cc:564] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [sys.catalog]: configured and running, proceeding with master startup.
11:49:08.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.083629 849 raft_consensus.cc:1261] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [term 1 FOLLOWER]: Refusing update from remote peer d822ebbc241f4649bf7b949f02457dda: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
11:49:08.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.097515 1040 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [LEADER]: Connected to new peer: Peer: permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
11:49:08.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.114226 830 tablet.cc:2369] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:49:08.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.224261 1089 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d822ebbc241f4649bf7b949f02457dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:08.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.223964 1080 sys_catalog.cc:455] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [sys.catalog]: SysCatalogTable state changed. Reason: New leader d822ebbc241f4649bf7b949f02457dda. Latest consensus state: current_term: 1 leader_uuid: "d822ebbc241f4649bf7b949f02457dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:08.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.225970 1080 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [sys.catalog]: This master's current role is: FOLLOWER
11:49:08.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.225530 1089 sys_catalog.cc:458] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f [sys.catalog]: This master's current role is: FOLLOWER
11:49:08.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.258819 1096 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
11:49:08.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.564222 1078 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:08.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.564927 1078 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:08.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.565686 1078 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:08.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.611716 1069 catalog_manager.cc:1259] Loaded cluster ID: 356c71ce9da14ec4b306ddf6971f6e77
11:49:08.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.612603 1069 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8: loading cluster ID for follower catalog manager: success
11:49:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.621243 1078 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:08.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.621869 1069 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8: acquiring CA information for follower catalog manager: success
11:49:08.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.623024 1078 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.199.1
11:49:08.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.629025 1069 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d2016a02f5314e1d87b335b60d0d05a8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:49:08.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.686966 1078 tablet_server_runner.cc:78] Tablet server non-default flags:
11:49:08.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:08.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:08.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:08.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:08.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/data
11:49:08.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/wal
11:49:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.1:0
11:49:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:08.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/data/info.pb
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.1
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.199.1
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/logs
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:08.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.688637 1078 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:08.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.690861 1078 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:08.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.717937 1103 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.742987 1040 sys_catalog.cc:455] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d822ebbc241f4649bf7b949f02457dda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b4f03826fba74f27b10c47d56ba2312f" member_type: VOTER last_known_addr { host: "127.0.199.62" port: 34729 } } peers { permanent_uuid: "d2016a02f5314e1d87b335b60d0d05a8" member_type: VOTER last_known_addr { host: "127.0.199.61" port: 37391 } } peers { permanent_uuid: "d822ebbc241f4649bf7b949f02457dda" member_type: VOTER last_known_addr { host: "127.0.199.60" port: 41761 } } }
11:49:08.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:08.745568 1040 sys_catalog.cc:458] T 00000000000000000000000000000000 P d822ebbc241f4649bf7b949f02457dda [sys.catalog]: This master's current role is: LEADER
11:49:09.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:09.291221 1096 catalog_manager.cc:1259] Loaded cluster ID: 356c71ce9da14ec4b306ddf6971f6e77
11:49:09.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:09.306202 1096 catalog_manager.cc:1552] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f: loading cluster ID for follower catalog manager: success
11:49:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:09.366295 1096 catalog_manager.cc:1574] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f: acquiring CA information for follower catalog manager: success
11:49:09.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:09.401852 1096 catalog_manager.cc:1602] T 00000000000000000000000000000000 P b4f03826fba74f27b10c47d56ba2312f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
11:49:10.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.724860 1105 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:10.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.461293 1078 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:49:10.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:10.461872 1104 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:49:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:08.720074 1102 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:10.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.467110 1078 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.477416 1078 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:10.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.479315 1078 hybrid_clock.cc:648] HybridClock initialized: now 1781869750479235 us; error 97 us; skew 500 ppm
11:49:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.480832 1078 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:10.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.508927 1078 webserver.cc:460] Webserver started at http://127.0.199.1:41267/ using document root <none> and password file <none>
11:49:10.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.512217 1078 fs_manager.cc:359] Metadata directory not provided
11:49:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.512846 1078 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:10.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.513646 1078 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.522709 1078 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/data/instance:
11:49:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e6ee6be56fc402083446f22ca3560c5"
11:49:10.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:10 on dist-test-slave-kqf6"
11:49:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.525162 1078 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/wal/instance:
11:49:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e6ee6be56fc402083446f22ca3560c5"
11:49:10.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:10 on dist-test-slave-kqf6"
11:49:10.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.541626 1078 fs_manager.cc:691] Time spent creating directory manager: real 0.015s user 0.009s sys 0.004s
11:49:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.555895 1118 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:10.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.557633 1078 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.003s sys 0.007s
11:49:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.558270 1078 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/wal
11:49:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1e6ee6be56fc402083446f22ca3560c5"
11:49:10.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:10 on dist-test-slave-kqf6"
11:49:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.558984 1078 fs_report.cc:383] FS layout report
11:49:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/wal
11:49:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/wal
11:49:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/data/data
11:49:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.616540 1078 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:10.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.618494 1078 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:10.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.619055 1078 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.622818 1078 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:49:10.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.629284 1078 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:49:10.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.629593 1078 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:49:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.629886 1078 ts_tablet_manager.cc:610] Registered 0 tablets
11:49:10.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.630088 1078 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:49:10.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.859922 1078 rpc_server.cc:307] RPC server started. Bound to: 127.0.199.1:43637
11:49:10.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.860033 1230 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.199.1:43637 every 8 connection(s)
11:49:10.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.863494 1078 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/data/info.pb
11:49:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.868192 796 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1078
11:49:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.868827 796 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-0/wal/instance
11:49:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:10.898699 796 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/wal
11:49:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/data
11:49:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:10.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:10.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:10.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/logs
11:49:10.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/data/info.pb
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.2:0
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.199.2
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.2
11:49:10.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:10.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:49:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.023236 1238 heartbeater.cc:347] Connected to a master server at 127.0.199.62:34729
11:49:11.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.023734 1238 heartbeater.cc:464] Registering TS with master...
11:49:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.024484 1232 heartbeater.cc:347] Connected to a master server at 127.0.199.61:37391
11:49:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.024827 1232 heartbeater.cc:464] Registering TS with master...
11:49:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.025005 1238 heartbeater.cc:511] Master 127.0.199.62:34729 requested a full tablet report, sending...
11:49:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.025784 1232 heartbeater.cc:511] Master 127.0.199.61:37391 requested a full tablet report, sending...
11:49:11.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.030247 841 ts_manager.cc:194] Registered new tserver with Master: 1e6ee6be56fc402083446f22ca3560c5 (127.0.199.1:43637)
11:49:11.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.031621 1231 heartbeater.cc:347] Connected to a master server at 127.0.199.60:41761
11:49:11.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.031530 916 ts_manager.cc:194] Registered new tserver with Master: 1e6ee6be56fc402083446f22ca3560c5 (127.0.199.1:43637)
11:49:11.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.031986 1231 heartbeater.cc:464] Registering TS with master...
11:49:11.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.032882 1231 heartbeater.cc:511] Master 127.0.199.60:41761 requested a full tablet report, sending...
11:49:11.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.035830 996 ts_manager.cc:194] Registered new tserver with Master: 1e6ee6be56fc402083446f22ca3560c5 (127.0.199.1:43637)
11:49:11.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.039637 996 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.199.1:37699
11:49:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:11.441637 1241 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:11.442153 1241 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:11.442639 1241 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:11.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:11.476954 1241 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:11.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:11.477974 1241 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.199.2
11:49:11.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.515424 1241 tablet_server_runner.cc:78] Tablet server non-default flags:
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/data
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/wal
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.2:0
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/data/info.pb
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.2
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.199.2
11:49:11.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/logs
11:49:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.517045 1241 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:11.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:11.519124 1241 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:11.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:11.543376 1252 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:12.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:12.045243 1231 heartbeater.cc:503] Master 127.0.199.60:41761 was elected leader, sending a full tablet report...
11:49:13.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:11.543627 1253 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:13.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:11.546527 1255 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.111053 1241 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:49:13.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.110869 1254 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1565 milliseconds
11:49:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.113363 1241 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.116357 1241 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:13.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.117892 1241 hybrid_clock.cc:648] HybridClock initialized: now 1781869753117961 us; error 167 us; skew 500 ppm
11:49:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.118912 1241 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:13.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.127585 1241 webserver.cc:460] Webserver started at http://127.0.199.2:38965/ using document root <none> and password file <none>
11:49:13.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.128906 1241 fs_manager.cc:359] Metadata directory not provided
11:49:13.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.129174 1241 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:13.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.129719 1241 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:13.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.134886 1241 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/data/instance:
11:49:13.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a2b1893ccd54145b5d589d75f2efb60"
11:49:13.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:13 on dist-test-slave-kqf6"
11:49:13.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.135993 1241 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/wal/instance:
11:49:13.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a2b1893ccd54145b5d589d75f2efb60"
11:49:13.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:13 on dist-test-slave-kqf6"
11:49:13.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.143970 1241 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
11:49:13.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.150338 1263 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.151499 1241 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.151815 1241 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/wal
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9a2b1893ccd54145b5d589d75f2efb60"
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:13 on dist-test-slave-kqf6"
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.152124 1241 fs_report.cc:383] FS layout report
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/wal
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/wal
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/data/data
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:13.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:13.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.190860 1241 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:13.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.192487 1241 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.193017 1241 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:13.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.196637 1241 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:49:13.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.201294 1241 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:49:13.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.201500 1241 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:49:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.201838 1241 ts_tablet_manager.cc:610] Registered 0 tablets
11:49:13.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.202054 1241 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:49:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.364310 1241 rpc_server.cc:307] RPC server started. Bound to: 127.0.199.2:39009
11:49:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.364488 1375 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.199.2:39009 every 8 connection(s)
11:49:13.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.366786 1241 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/data/info.pb
11:49:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.375099 796 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1241
11:49:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.375887 796 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-1/wal/instance
11:49:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.390296 796 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/wal
11:49:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/data
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/logs
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/data/info.pb
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.3:0
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.199.3
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.3
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
11:49:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.423918 1376 heartbeater.cc:347] Connected to a master server at 127.0.199.60:41761
11:49:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.424499 1376 heartbeater.cc:464] Registering TS with master...
11:49:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.425926 1376 heartbeater.cc:511] Master 127.0.199.60:41761 requested a full tablet report, sending...
11:49:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.429388 1377 heartbeater.cc:347] Connected to a master server at 127.0.199.61:37391
11:49:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.429752 1377 heartbeater.cc:464] Registering TS with master...
11:49:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.429756 996 ts_manager.cc:194] Registered new tserver with Master: 9a2b1893ccd54145b5d589d75f2efb60 (127.0.199.2:39009)
11:49:13.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.430509 1377 heartbeater.cc:511] Master 127.0.199.61:37391 requested a full tablet report, sending...
11:49:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.433312 916 ts_manager.cc:194] Registered new tserver with Master: 9a2b1893ccd54145b5d589d75f2efb60 (127.0.199.2:39009)
11:49:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.433413 996 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.199.2:49465
11:49:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.439167 1378 heartbeater.cc:347] Connected to a master server at 127.0.199.62:34729
11:49:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.439561 1378 heartbeater.cc:464] Registering TS with master...
11:49:13.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.440819 1378 heartbeater.cc:511] Master 127.0.199.62:34729 requested a full tablet report, sending...
11:49:13.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.443481 841 ts_manager.cc:194] Registered new tserver with Master: 9a2b1893ccd54145b5d589d75f2efb60 (127.0.199.2:39009)
11:49:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.741873 1386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.742375 1386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:13.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.742825 1386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.775882 1386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.776883 1386 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.199.3
11:49:13.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.813359 1386 tablet_server_runner.cc:78] Tablet server non-default flags:
11:49:13.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:13.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.199.20:39207
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/data
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/wal
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.199.3:0
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/data/info.pb
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.199.3
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.199.62:34729,127.0.199.61:37391,127.0.199.60:41761
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.199.3
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/logs
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.814615 1386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:13.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.816298 1386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.836045 1393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:14.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:14.438758 1376 heartbeater.cc:503] Master 127.0.199.60:41761 was elected leader, sending a full tablet report...
11:49:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.836941 1394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:13.840075 1386 server_base.cc:1034] running on GCE node
11:49:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:13.837494 1396 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.135967 1386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:15.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.138856 1386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:15.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.140321 1386 hybrid_clock.cc:648] HybridClock initialized: now 1781869755140272 us; error 63 us; skew 500 ppm
11:49:15.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.141323 1386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:15.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.148473 1386 webserver.cc:460] Webserver started at http://127.0.199.3:35467/ using document root <none> and password file <none>
11:49:15.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.149457 1386 fs_manager.cc:359] Metadata directory not provided
11:49:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.149653 1386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.150024 1386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:15.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.155525 1386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/data/instance:
11:49:15.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c47d7d27124f40dd890b7f0b24d3c5d3"
11:49:15.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:15 on dist-test-slave-kqf6"
11:49:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.156726 1386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/wal/instance:
11:49:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c47d7d27124f40dd890b7f0b24d3c5d3"
11:49:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:15 on dist-test-slave-kqf6"
11:49:15.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.163579 1386 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
11:49:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.169752 1403 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.171164 1386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
11:49:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.171445 1386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/wal
11:49:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c47d7d27124f40dd890b7f0b24d3c5d3"
11:49:15.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:15 on dist-test-slave-kqf6"
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.171809 1386 fs_report.cc:383] FS layout report
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/wal
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/wal
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/data/data
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:15.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.217084 1386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:15.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.218564 1386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:15.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.218991 1386 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:15.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.221693 1386 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
11:49:15.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.226404 1386 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
11:49:15.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.226603 1386 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
11:49:15.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.226800 1386 ts_tablet_manager.cc:610] Registered 0 tablets
11:49:15.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.226930 1386 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
11:49:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.401145 1386 rpc_server.cc:307] RPC server started. Bound to: 127.0.199.3:36285
11:49:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.401261 1516 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.199.3:36285 every 8 connection(s)
11:49:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.403607 1386 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/data/info.pb
11:49:15.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.404560 796 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1386
11:49:15.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.405028 796 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster7913874277588147942/ts-2/wal/instance
11:49:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.453621 1517 heartbeater.cc:347] Connected to a master server at 127.0.199.60:41761
11:49:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.454121 1517 heartbeater.cc:464] Registering TS with master...
11:49:15.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.455330 1517 heartbeater.cc:511] Master 127.0.199.60:41761 requested a full tablet report, sending...
11:49:15.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.455960 1519 heartbeater.cc:347] Connected to a master server at 127.0.199.62:34729
11:49:15.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.456226 1519 heartbeater.cc:464] Registering TS with master...
11:49:15.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.457010 1519 heartbeater.cc:511] Master 127.0.199.62:34729 requested a full tablet report, sending...
11:49:15.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.458120 996 ts_manager.cc:194] Registered new tserver with Master: c47d7d27124f40dd890b7f0b24d3c5d3 (127.0.199.3:36285)
11:49:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.458973 1518 heartbeater.cc:347] Connected to a master server at 127.0.199.61:37391
11:49:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.459273 1518 heartbeater.cc:464] Registering TS with master...
11:49:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.459775 841 ts_manager.cc:194] Registered new tserver with Master: c47d7d27124f40dd890b7f0b24d3c5d3 (127.0.199.3:36285)
11:49:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.460088 1518 heartbeater.cc:511] Master 127.0.199.61:37391 requested a full tablet report, sending...
11:49:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.460413 996 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.199.3:39217
11:49:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.462443 916 ts_manager.cc:194] Registered new tserver with Master: c47d7d27124f40dd890b7f0b24d3c5d3 (127.0.199.3:36285)
11:49:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:15.480314 796 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
11:49:15.491 [DEBUG - main] (MiniKuduCluster.java:195) Response:
11:49:15.492 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
11:49:15.504 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.199.62"
port: 34729
}
bound_http_address {
host: "127.0.199.62"
port: 38929
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.199.61"
port: 37391
}
bound_http_address {
host: "127.0.199.61"
port: 40615
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.199.60"
port: 41761
}
bound_http_address {
host: "127.0.199.60"
port: 43199
}
}
}
11:49:15.509 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:49:15.512 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.199.1"
port: 43637
}
bound_http_address {
host: "127.0.199.1"
port: 41267
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.199.2"
port: 39009
}
bound_http_address {
host: "127.0.199.2"
port: 38965
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.199.3"
port: 36285
}
bound_http_address {
host: "127.0.199.3"
port: 35467
}
}
}
11:49:15.512 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
11:49:15.707 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
11:49:15.830 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task0xMXX5/test-tmp
11:49:15.831 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
11:49:15.831 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task0xMXX5/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
11:49:15.835 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
11:49:16.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:16.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
11:49:16.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:16.257246 1528 mini_oidc.cc:152] Starting JWKS server
11:49:16.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:16.261022 1528 webserver.cc:460] Webserver started at http://127.0.0.1:38725/ using document root <none> and password file <none>
11:49:16.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:16.261477 1528 mini_oidc.cc:189] Starting OIDC Discovery server
11:49:16.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:16.263638 1528 webserver.cc:460] Webserver started at http://127.0.0.1:35425/ using document root <none> and password file <none>
11:49:16.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
11:49:16.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/principal' for realm 'KRBTEST.COM',
11:49:16.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
11:49:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:16 dist-test-slave-kqf6 krb5kdc[1540](info): setting up network...
11:49:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
11:49:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:16 dist-test-slave-kqf6 krb5kdc[1540](info): set up 2 sockets
11:49:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:16 dist-test-slave-kqf6 krb5kdc[1540](info): commencing operation
11:49:16.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
11:49:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:16.464417 1517 heartbeater.cc:503] Master 127.0.199.60:41761 was elected leader, sending a full tablet report...
11:49:17.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.415508 1528 mini_kdc.cc:121] Time spent starting KDC: real 1.123s user 0.003s sys 0.017s
11:49:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
11:49:17.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:49:17.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
11:49:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
11:49:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:49:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
11:49:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
11:49:17.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:49:17.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
11:49:17.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
11:49:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/test-user.keytab.
11:49:17.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/test-user.keytab.
11:49:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:17 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869757, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:49:17.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T11:49:17Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
11:49:17.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T11:49:17Z Disabled control of system clock
11:49:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.126.62@KRBTEST.COM; defaulting to no policy
11:49:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:49:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.126.62@KRBTEST.COM" created.
11:49:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:49:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.126.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.62.keytab.
11:49:17.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.126.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.62.keytab.
11:49:17.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:17.617527 1528 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/logs
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/info.pb
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:17.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.62:37119
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.62
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.62.keytab
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.62
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:49:17.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.943503 1553 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:17.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.944095 1553 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:17.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.944713 1553 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.977269 1553 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:49:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.977556 1553 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.977839 1553 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.978027 1553 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.978200 1553 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.978368 1553 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:17.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:17.978525 1553 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38725/jwks.json
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:18.015873 1553 master_runner.cc:386] Master server non-default flags:
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.62.keytab
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.62
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:18.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.62:37119
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/info.pb
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.62
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/logs
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:18.017130 1553 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:18.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:18.018075 1553 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:18.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:18.041353 1560 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:19.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:18.043525 1559 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:19.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:18.041399 1562 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:19.414669 1561 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1370 milliseconds
11:49:19.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.414803 1553 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:49:19.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.416251 1553 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:19.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.419010 1553 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:19.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.420651 1553 hybrid_clock.cc:648] HybridClock initialized: now 1781869759420635 us; error 62 us; skew 500 ppm
11:49:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.421552 1553 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:19.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:19 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869759, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.428151 1553 init.cc:377] Logged in from keytab as kudu/127.1.126.62@KRBTEST.COM (short username kudu)
11:49:19.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.434444 1553 webserver.cc:460] Webserver started at http://127.1.126.62:40745/ using document root <none> and password file <none>
11:49:19.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.435370 1553 fs_manager.cc:359] Metadata directory not provided
11:49:19.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.435578 1553 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:19.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.435961 1553 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:19.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.441421 1553 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/instance:
11:49:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae"
11:49:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:19 on dist-test-slave-kqf6"
11:49:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.442525 1553 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal/instance:
11:49:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae"
11:49:19.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:19 on dist-test-slave-kqf6"
11:49:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.451771 1553 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.009s sys 0.000s
11:49:19.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.459002 1571 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:19.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.461012 1553 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.002s
11:49:19.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.461721 1553 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:19.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae"
11:49:19.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:19 on dist-test-slave-kqf6"
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.462235 1553 fs_report.cc:383] FS layout report
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/data
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:19.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.505316 1553 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:19.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.512470 1553 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:19.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.512943 1553 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.600193 1553 rpc_server.cc:307] RPC server started. Bound to: 127.1.126.62:37119
11:49:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.600235 1622 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.126.62:37119 every 8 connection(s)
11:49:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.603317 1553 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/info.pb
11:49:19.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.610277 1623 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:49:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.612929 1528 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1553
11:49:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.613453 1528 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal/instance
11:49:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.621814 1623 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } has no permanent_uuid. Determining permanent_uuid...
11:49:19.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.126.61@KRBTEST.COM; defaulting to no policy
11:49:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:49:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.126.61@KRBTEST.COM" created.
11:49:19.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:19 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869759, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.62@KRBTEST.COM for kudu/127.1.126.62@KRBTEST.COM
11:49:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:49:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.126.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.61.keytab.
11:49:19.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.126.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.61.keytab.
11:49:19.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.670456 1528 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/logs
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/info.pb
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:19.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.61:43799
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.61
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.61.keytab
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.61
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:49:19.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.671994 1623 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } has no permanent_uuid. Determining permanent_uuid...
11:49:19.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:19.674479 1575 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:19.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:19.678941 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.61:43799: Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:19.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.728530 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } attempt: 1
11:49:19.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:19.732234 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.61:43799: Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:19.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.800643 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } attempt: 2
11:49:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:19.804885 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.61:43799: Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:19.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:19.896375 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } attempt: 3
11:49:19.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:19.901149 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.61:43799: Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.006079 1629 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:20.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.007160 1629 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:20.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.008064 1629 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.044183 1629 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:49:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.044529 1629 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:20.044611 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } attempt: 4
11:49:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.044971 1629 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.045356 1629 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:20.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.045753 1629 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:20.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.046066 1629 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:20.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.046464 1629 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38725/jwks.json
11:49:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.049068 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.61:43799: Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:20.098869 1629 master_runner.cc:386] Master server non-default flags:
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.61.keytab
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.61
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.61:43799
11:49:20.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/info.pb
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.61
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/logs
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:20.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:20.100103 1629 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:20.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:20.101505 1629 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.125599 1637 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:20.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:20.348567 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } attempt: 5
11:49:20.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.352118 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.61:43799: Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:20.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:20.899542 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } attempt: 6
11:49:20.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.906821 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.61:43799: Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:21.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:21.354171 1227 debug-util.cc:398] Leaking SignalData structure 0x7b08000afda0 after lost signal to thread 1097
11:49:21.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:21.355046 1227 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4fe0 after lost signal to thread 1230
11:49:21.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.127377 1639 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:21.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:20.128119 1629 server_base.cc:1034] running on GCE node
11:49:21.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:20.126381 1636 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.545413 1629 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.550233 1629 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:21.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.552053 1629 hybrid_clock.cc:648] HybridClock initialized: now 1781869761551990 us; error 46 us; skew 500 ppm
11:49:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.553375 1629 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:21.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:21 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:21.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.566262 1629 init.cc:377] Logged in from keytab as kudu/127.1.126.61@KRBTEST.COM (short username kudu)
11:49:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.572785 1629 webserver.cc:460] Webserver started at http://127.1.126.61:35481/ using document root <none> and password file <none>
11:49:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.573752 1629 fs_manager.cc:359] Metadata directory not provided
11:49:21.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.573956 1629 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:21.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.574347 1629 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.580706 1629 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/instance:
11:49:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4407124148eb4423b5862adc784792b5"
11:49:21.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:21 on dist-test-slave-kqf6"
11:49:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.582163 1629 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal/instance:
11:49:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4407124148eb4423b5862adc784792b5"
11:49:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:21 on dist-test-slave-kqf6"
11:49:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.591698 1629 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.005s sys 0.004s
11:49:21.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.598961 1651 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:21.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.600605 1629 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
11:49:21.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.601240 1629 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:21.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4407124148eb4423b5862adc784792b5"
11:49:21.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:21 on dist-test-slave-kqf6"
11:49:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.601954 1629 fs_report.cc:383] FS layout report
11:49:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/data
11:49:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:21.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:21.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.660478 1629 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.670513 1629 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:21.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.671226 1629 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.779676 1629 rpc_server.cc:307] RPC server started. Bound to: 127.1.126.61:43799
11:49:21.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.779742 1703 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.126.61:43799 every 8 connection(s)
11:49:21.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.782616 1629 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/info.pb
11:49:21.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.790239 1528 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1629
11:49:21.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.790522 1704 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:49:21.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.790788 1528 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal/instance
11:49:21.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.126.60@KRBTEST.COM; defaulting to no policy
11:49:21.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.810209 1704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } has no permanent_uuid. Determining permanent_uuid...
11:49:21.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:21.967340 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } attempt: 7
11:49:21.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:49:21.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.126.60@KRBTEST.COM" created.
11:49:21.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:21 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.61@KRBTEST.COM for kudu/127.1.126.62@KRBTEST.COM
11:49:21.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:21 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869759, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.62@KRBTEST.COM for kudu/127.1.126.61@KRBTEST.COM
11:49:22.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.003285 1704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } has no permanent_uuid. Determining permanent_uuid...
11:49:22.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.011157 1623 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } has no permanent_uuid. Determining permanent_uuid...
11:49:22.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.019637 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.078091 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 1
11:49:22.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.082613 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.164105 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 2
11:49:22.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.168833 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:22 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869761, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.61@KRBTEST.COM for kudu/127.1.126.61@KRBTEST.COM
11:49:22.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.205164 1704 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } has no permanent_uuid. Determining permanent_uuid...
11:49:22.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.207993 1655 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.212139 1704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.254333 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 3
11:49:22.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.258301 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.261683 1704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 1
11:49:22.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.266624 1704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.335095 1704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 2
11:49:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.339219 1704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.398694 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 4
11:49:22.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.403230 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.430671 1704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 3
11:49:22.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.434721 1704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.578212 1704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 4
11:49:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.583222 1704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.686650 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 5
11:49:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.691185 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:22.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:22.882758 1704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 5
11:49:22.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:22.887192 1704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:23.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
11:49:23.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.126.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.60.keytab.
11:49:23.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.126.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.60.keytab.
11:49:23.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.237699 1528 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.1.126.60: real 1.445s user 0.003s sys 0.012s
11:49:23.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:23.239569 1528 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/logs
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/info.pb
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.60:36889
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.60
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.60.keytab
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.60
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:49:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:49:23.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:23.243847 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 6
11:49:23.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.252349 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:23.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:23.434635 1704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 6
11:49:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.442723 1704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.608034 1715 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.608647 1715 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:23.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.609236 1715 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.648078 1715 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:49:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.648386 1715 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.648754 1715 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.648965 1715 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.649168 1715 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.649349 1715 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:23.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.649495 1715 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38725/jwks.json
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:23.694097 1715 master_runner.cc:386] Master server non-default flags:
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.60.keytab
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.60
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.60:36889
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:23.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/info.pb
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.60
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/logs
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:23.695276 1715 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:23.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:23.696280 1715 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:23.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.716254 1729 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:24.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:24.285846 1623 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 7
11:49:24.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:24.292074 1623 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:24.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:24.503234 1704 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } attempt: 7
11:49:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:24.515298 1704 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.126.60:36889: Network error: Client connection negotiation failed: client connection to 127.1.126.60:36889: connect: Connection refused (error 111)
11:49:25.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.716284 1726 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:25.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:23.717449 1727 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.003984 1728 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:49:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.004171 1715 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:49:25.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.010094 1715 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:25.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.013361 1715 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:25.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.015302 1715 hybrid_clock.cc:648] HybridClock initialized: now 1781869765015257 us; error 55 us; skew 500 ppm
11:49:25.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.016274 1715 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:25.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869765, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.023043 1715 init.cc:377] Logged in from keytab as kudu/127.1.126.60@KRBTEST.COM (short username kudu)
11:49:25.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.029228 1715 webserver.cc:460] Webserver started at http://127.1.126.60:42161/ using document root <none> and password file <none>
11:49:25.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.030352 1715 fs_manager.cc:359] Metadata directory not provided
11:49:25.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.030579 1715 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.031029 1715 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
11:49:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.036957 1715 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/instance:
11:49:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc1e8e3ed3514234ba472ac83b7ea41b"
11:49:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:25 on dist-test-slave-kqf6"
11:49:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.038548 1715 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal/instance:
11:49:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc1e8e3ed3514234ba472ac83b7ea41b"
11:49:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:25 on dist-test-slave-kqf6"
11:49:25.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.049791 1715 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.007s sys 0.005s
11:49:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.058318 1743 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.059828 1715 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
11:49:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.060343 1715 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc1e8e3ed3514234ba472ac83b7ea41b"
11:49:25.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:25 on dist-test-slave-kqf6"
11:49:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.060750 1715 fs_report.cc:383] FS layout report
11:49:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/data
11:49:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:25.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:25.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.104688 1715 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.111353 1715 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.111815 1715 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:25.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.196929 1715 rpc_server.cc:307] RPC server started. Bound to: 127.1.126.60:36889
11:49:25.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.196995 1794 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.126.60:36889 every 8 connection(s)
11:49:25.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.200696 1715 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/info.pb
11:49:25.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.208078 1795 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
11:49:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.208968 1528 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1715
11:49:25.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.209447 1528 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal/instance
11:49:25.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.228223 1795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } has no permanent_uuid. Determining permanent_uuid...
11:49:25.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869765, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.60@KRBTEST.COM for kudu/127.1.126.62@KRBTEST.COM
11:49:25.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869757, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.126.60@KRBTEST.COM
11:49:25.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.272222 1795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } has no permanent_uuid. Determining permanent_uuid...
11:49:25.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869765, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.60@KRBTEST.COM for kudu/127.1.126.61@KRBTEST.COM
11:49:25.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869757, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.126.61@KRBTEST.COM
11:49:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.301936 1795 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } has no permanent_uuid. Determining permanent_uuid...
11:49:25.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869765, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.60@KRBTEST.COM for kudu/127.1.126.60@KRBTEST.COM
11:49:25.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869757, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.126.62@KRBTEST.COM
11:49:25.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.337819 1795 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: Bootstrap starting.
11:49:25.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.340939 1528 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
11:49:25.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.344933 1795 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: Neither blocks nor log segments found. Creating new log.
11:49:25.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.346657 1795 log.cc:824] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: Log is configured to *not* fsync() on all Append() calls
11:49:25.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:25.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
11:49:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.352130 1795 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: No bootstrap required, opened a new log
11:49:25.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
11:49:25.367 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
11:49:25.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.126.62"
port: 37119
}
bound_http_address {
host: "127.1.126.62"
port: 40745
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.126.61"
port: 43799
}
bound_http_address {
host: "127.1.126.61"
port: 35481
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.126.60"
port: 36889
}
bound_http_address {
host: "127.1.126.60"
port: 42161
}
}
}
11:49:25.371 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
11:49:25.372 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:25.372 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:49:25.374 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:49:25.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.386927 1795 raft_consensus.cc:348] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:25.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.387893 1795 raft_consensus.cc:374] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:49:25.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.388223 1795 raft_consensus.cc:729] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc1e8e3ed3514234ba472ac83b7ea41b, State: Initialized, Role: FOLLOWER
11:49:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.389295 1795 consensus_queue.cc:260] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:25.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.392532 1805 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } } }
11:49:25.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.394034 1805 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [sys.catalog]: This master's current role is: FOLLOWER
11:49:25.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.394166 1795 sys_catalog.cc:564] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [sys.catalog]: configured and running, proceeding with master startup.
11:49:25.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869765, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:25.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:49:25.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:25.424 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc
11:49:25.432 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:49:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.433068 1818 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.433470 1818 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:25.433 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc
11:49:25.435 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:49:25.439 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
11:49:25.442 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
11:49:25.442 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
11:49:25.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.426683 1715 master_runner.cc:186] Error getting master registration for 127.1.126.62:37119: OK, instance_id { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" instance_seqno: 1781869759575024 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
11:49:25.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.471961 1715 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.474229 1805 raft_consensus.cc:482] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:25.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.474922 1805 raft_consensus.cc:504] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:25.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.478248 1805 leader_election.cc:288] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119), 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799)
11:49:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:25 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869765, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
11:49:25.495 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:25.495 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc
11:49:25.497 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
11:49:25.497 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.126.61:43799
11:49:25.499 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
11:49:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.499822 1528 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 1629
11:49:25.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.532861 1747 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.1.126.61:43799 (error 108)
11:49:25.534 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:25.534 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.126.62:37119
11:49:25.535 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
11:49:25.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.535341 1528 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 1553
11:49:25.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.547032 1747 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799): Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:25.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:25.563 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.126.60:36889
11:49:25.563 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
11:49:25.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.564230 1528 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 1715
11:49:25.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:25.587 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.126.61:43799
11:49:25.589 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:25.590734 1528 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/logs
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/info.pb
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.61:43799
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.61
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35481
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.61.keytab
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.61
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:25.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:25.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:49:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.925489 1826 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:25.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.926179 1826 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:25.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.927006 1826 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:25.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.961052 1826 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:49:25.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.961398 1826 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.961812 1826 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.962138 1826 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.962443 1826 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:25.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.962745 1826 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:25.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:25.962991 1826 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38725/jwks.json
11:49:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:26.001363 1826 master_runner.cc:386] Master server non-default flags:
11:49:26.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.61.keytab
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.61
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.61:43799
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/info.pb
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.61
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35481
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:26.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/logs
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:26.002894 1826 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:26.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:26.004079 1826 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:26.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:26.023818 1832 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:26.023811 1833 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:26.024250 1835 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.293083 1834 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:49:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.293190 1826 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:49:27.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.297770 1826 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:27.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.300733 1826 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.302165 1826 hybrid_clock.cc:648] HybridClock initialized: now 1781869767302131 us; error 45 us; skew 500 ppm
11:49:27.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.303073 1826 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:27.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:27 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869767, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:27.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.314808 1826 init.cc:377] Logged in from keytab as kudu/127.1.126.61@KRBTEST.COM (short username kudu)
11:49:27.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.319916 1826 webserver.cc:460] Webserver started at http://127.1.126.61:35481/ using document root <none> and password file <none>
11:49:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.321094 1826 fs_manager.cc:359] Metadata directory not provided
11:49:27.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.321306 1826 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.330049 1826 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.000s
11:49:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.335112 1844 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:27.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.336316 1826 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.336769 1826 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4407124148eb4423b5862adc784792b5"
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:21 on dist-test-slave-kqf6"
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.339156 1826 fs_report.cc:383] FS layout report
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/data
11:49:27.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:27.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.392561 1826 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:27.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.399183 1826 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:27.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.399859 1826 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:27.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.477491 1826 rpc_server.cc:307] RPC server started. Bound to: 127.1.126.61:43799
11:49:27.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.477692 1895 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.126.61:43799 every 8 connection(s)
11:49:27.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.480384 1826 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/data/info.pb
11:49:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.484748 1528 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1826
11:49:27.485 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:27.485 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.126.62:37119
11:49:27.486 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
11:49:27.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.487883 1528 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:27.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:27.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:27.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data
11:49:27.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/logs
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/info.pb
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.62:37119
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.62
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40745
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.62.keytab
11:49:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.62
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:27.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:49:27.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.494963 1896 sys_catalog.cc:263] Verifying existing consensus state
11:49:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 11:49:27.495720 1896 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:49:27.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.496296 1826 master.cc:537] Master@127.1.126.61:43799 shutting down...
11:49:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.565683 1826 master.cc:552] Master@127.1.126.61:43799 shutdown complete.
11:49:27.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:49:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.817934 1898 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:27.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.818538 1898 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:27.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.819180 1898 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.852034 1898 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:49:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.852267 1898 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.852618 1898 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.852829 1898 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:27.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.853012 1898 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:27.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.853204 1898 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:27.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.853343 1898 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38725/jwks.json
11:49:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.894565 1898 master_runner.cc:386] Master server non-default flags:
11:49:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.62.keytab
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.62
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.62:37119
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/info.pb
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.62
11:49:27.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40745
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/logs
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.895908 1898 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:27.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:27.897168 1898 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:27.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.915867 1904 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:29.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.915917 1907 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:29.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:27.916190 1905 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:29.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.245700 1906 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:49:29.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.245822 1898 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:49:29.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.250309 1898 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.252851 1898 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.254235 1898 hybrid_clock.cc:648] HybridClock initialized: now 1781869769254199 us; error 61 us; skew 500 ppm
11:49:29.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.255163 1898 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:29.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:29 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869769, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.261366 1898 init.cc:377] Logged in from keytab as kudu/127.1.126.62@KRBTEST.COM (short username kudu)
11:49:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.267366 1898 webserver.cc:460] Webserver started at http://127.1.126.62:40745/ using document root <none> and password file <none>
11:49:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.268492 1898 fs_manager.cc:359] Metadata directory not provided
11:49:29.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.268764 1898 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:29.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.277285 1898 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
11:49:29.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.281847 1916 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:29.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.283121 1898 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
11:49:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.283514 1898 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae"
11:49:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:19 on dist-test-slave-kqf6"
11:49:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.286084 1898 fs_report.cc:383] FS layout report
11:49:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:29.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/data
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:29.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.324095 1898 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.330224 1898 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:29.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.330754 1898 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.408818 1898 rpc_server.cc:307] RPC server started. Bound to: 127.1.126.62:37119
11:49:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.408999 1967 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.126.62:37119 every 8 connection(s)
11:49:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.411823 1898 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/data/info.pb
11:49:29.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.420331 1528 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1898
11:49:29.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:29.421 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.126.60:36889
11:49:29.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.423779 1528 external_mini_cluster.cc:1326] Running /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/logs
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/info.pb
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.60:36889
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.60
11:49:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42161
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.60.keytab
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.60
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
11:49:29.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.424338 1968 sys_catalog.cc:263] Verifying existing consensus state
11:49:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260619 11:49:29.425361 1968 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:49:29.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.425889 1898 master.cc:537] Master@127.1.126.62:37119 shutting down...
11:49:29.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.486495 1898 master.cc:552] Master@127.1.126.62:37119 shutdown complete.
11:49:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
11:49:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.757839 1970 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
11:49:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.758396 1970 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
11:49:29.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.758973 1970 flags.cc:425] Enabled unsafe flag: --never_fsync=true
11:49:29.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.791227 1970 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
11:49:29.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.791508 1970 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
11:49:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.791870 1970 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
11:49:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.792094 1970 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
11:49:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.792284 1970 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
11:49:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.792505 1970 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
11:49:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.792655 1970 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38725/jwks.json
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.832613 1970 master_runner.cc:386] Master server non-default flags:
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.126.20:42233
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.126.62:37119,127.1.126.61:43799,127.1.126.60:36889
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/krb5kdc/kudu_127.1.126.60.keytab
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.126.60
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.126.60:36889
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38725/jwks.json
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/info.pb
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.126.60
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42161
11:49:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
11:49:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
11:49:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
11:49:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
11:49:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
11:49:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/logs
11:49:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
11:49:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 19 Jun 2026 10:51:24 UTC on ccycloud.quasar-eairov.root.comops.site
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4455438
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.834009 1970 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
11:49:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:29.835062 1970 file_cache.cc:493] Constructed file cache file cache with capacity 419430
11:49:29.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.852553 1979 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.852583 1977 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:31.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:29.853394 1976 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
11:49:31.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:31.126506 1978 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
11:49:31.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.126703 1970 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
11:49:31.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.132236 1970 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
11:49:31.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.136330 1970 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
11:49:31.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.137899 1970 hybrid_clock.cc:648] HybridClock initialized: now 1781869771137861 us; error 51 us; skew 500 ppm
11:49:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.139102 1970 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
11:49:31.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:31 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869771, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:31.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.146490 1970 init.cc:377] Logged in from keytab as kudu/127.1.126.60@KRBTEST.COM (short username kudu)
11:49:31.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.152076 1970 webserver.cc:460] Webserver started at http://127.1.126.60:42161/ using document root <none> and password file <none>
11:49:31.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.155561 1970 fs_manager.cc:359] Metadata directory not provided
11:49:31.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.155889 1970 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
11:49:31.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.166635 1970 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
11:49:31.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.171667 1987 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
11:49:31.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.173002 1970 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
11:49:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.173341 1970 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data,/tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dc1e8e3ed3514234ba472ac83b7ea41b"
11:49:31.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-19 11:49:25 on dist-test-slave-kqf6"
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.175694 1970 fs_report.cc:383] FS layout report
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/wal
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/data
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
11:49:31.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
11:49:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.211704 1970 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
11:49:31.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.217844 1970 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
11:49:31.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.218274 1970 kserver.cc:163] Server-wide thread pool size limit: 3276
11:49:31.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.294708 1970 rpc_server.cc:307] RPC server started. Bound to: 127.1.126.60:36889
11:49:31.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.294889 2039 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.126.60:36889 every 8 connection(s)
11:49:31.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.297952 1970 server_base.cc:1166] Dumped server information to /tmp/dist-test-task0xMXX5/test-tmp/mini-kudu-cluster4507199090347312262/master-2/data/info.pb
11:49:31.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.302211 1528 external_mini_cluster.cc:1388] Started /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu as pid 1970
11:49:31.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
11:49:31.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.311878 2040 sys_catalog.cc:263] Verifying existing consensus state
11:49:31.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.320994 2040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: Bootstrap starting.
11:49:31.366 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:31.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.368327 2040 log.cc:824] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: Log is configured to *not* fsync() on all Append() calls
11:49:31.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.381944 2040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
11:49:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.382786 2040 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: Bootstrap complete.
11:49:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.423591 2040 raft_consensus.cc:348] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.427582 2040 raft_consensus.cc:374] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
11:49:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.427963 2040 raft_consensus.cc:729] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc1e8e3ed3514234ba472ac83b7ea41b, State: Initialized, Role: FOLLOWER
11:49:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.429142 2040 consensus_queue.cc:260] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.432549 2044 sys_catalog.cc:455] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } } }
11:49:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.433566 2044 sys_catalog.cc:458] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [sys.catalog]: This master's current role is: FOLLOWER
11:49:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.434573 2040 sys_catalog.cc:564] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [sys.catalog]: configured and running, proceeding with master startup.
11:49:31.457 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:31.458 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.126.60:36889(127.1.126.60:36889)] connecting to peer
11:49:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:31.476503 1993 tablet.cc:2369] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: Can't schedule compaction. Clean time has not been advanced past its initial value.
11:49:31.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:31.501581 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:31.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:31.501942 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.493294 1970 master_runner.cc:186] Error getting master registration for 127.1.126.62:37119: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111),
11:49:31.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.533303 1970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:31.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.533 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.534 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.536 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.537 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.540 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.126.60:36889(127.1.126.60:36889)] Successfully connected to peer
11:49:31.617 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
11:49:31.625 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.126.60
11:49:31.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:31 dist-test-slave-kqf6 krb5kdc[1540](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869765, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.126.60@KRBTEST.COM
11:49:31.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.868853 2044 raft_consensus.cc:482] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.869439 2044 raft_consensus.cc:504] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:31.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:31.874621 1991 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111)
11:49:31.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.876669 2044 leader_election.cc:288] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119), 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799)
11:49:31.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:31.881636 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119): Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111)
11:49:31.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:31.884552 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799): Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:31.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.884987 1991 leader_election.cc:302] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dc1e8e3ed3514234ba472ac83b7ea41b; no voters: 4407124148eb4423b5862adc784792b5, 4f82a717bbcd4a0d8a90d4ecdc63f3ae
11:49:31.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:31.885879 2044 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:49:31.886 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x94b10212, L:/127.0.0.1:42644 - R:/127.1.126.60:36889] using SASL/GSSAPI
11:49:31.904 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:31.909 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:31.911 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:31.915 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.915 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.916 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.921 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.921 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:31.948 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:31.949 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:31.951 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.952 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.952 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.954 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.955 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.956 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.959 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:31.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:31.970 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:31.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.971 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.972 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.974 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.974 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.975 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.977 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:31.988 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:31.989 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:31.990 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.991 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.992 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:31.995 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.995 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.995 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:31.997 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:32.008 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:32.009 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:32.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.014 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.014 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.015 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.017 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:32.068 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:32.069 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:32.072 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.072 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.073 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.075 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.075 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.076 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.078 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:32.108 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:32.110 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:32.111 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.111 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.113 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.118 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.118 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.119 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.121 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:32.228 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:32.229 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:32.231 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.231 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.232 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.235 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.235 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.236 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.237 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:32.329 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:32.330 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:32.331 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.331 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.332 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.335 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.335 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.336 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.338 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:32.505100 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:32.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:32.505367 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:32.768 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:32.769 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:32.770 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.771 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:32.772 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.772 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:32.775 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:33.288 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:33.290 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:33.291 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:33.291 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:33.291 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:33.296 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:33.296 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:33.297 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:33.299 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:33.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:33.389288 2064 raft_consensus.cc:482] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:33.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:33.389737 2064 raft_consensus.cc:504] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:33.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:33.391678 2064 leader_election.cc:288] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119), 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799)
11:49:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:33.399595 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119): Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111)
11:49:33.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:33.400229 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799): Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:33.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:33.400692 1991 leader_election.cc:302] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dc1e8e3ed3514234ba472ac83b7ea41b; no voters: 4407124148eb4423b5862adc784792b5, 4f82a717bbcd4a0d8a90d4ecdc63f3ae
11:49:33.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:33.401376 2064 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:49:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:33.508652 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:33.508909 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:34.511612 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:34.511874 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:35.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:35.514700 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:35.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:35.514961 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:36.054201 2071 raft_consensus.cc:482] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:36.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:36.054615 2071 raft_consensus.cc:504] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:36.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:36.056002 2071 leader_election.cc:288] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119), 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799)
11:49:36.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:36.062579 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119): Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111)
11:49:36.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:36.063199 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799): Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:36.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:36.063462 1991 leader_election.cc:302] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dc1e8e3ed3514234ba472ac83b7ea41b; no voters: 4407124148eb4423b5862adc784792b5, 4f82a717bbcd4a0d8a90d4ecdc63f3ae
11:49:36.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:36.063992 2071 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:49:36.488 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:36.490 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:36.491 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:36.491 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:36.492 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:36.492 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:36.492 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:36.493 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:36.495 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:36.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:36.518093 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:36.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:36.518386 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:36.535633 1970 master_runner.cc:186] Error getting master registration for 127.1.126.62:37119: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111),
11:49:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:36.536031 1970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:37.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:37.521236 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:37.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:37.521497 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:38.205775 2076 raft_consensus.cc:482] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:38.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:38.206146 2076 raft_consensus.cc:504] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:38.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:38.207815 2076 leader_election.cc:288] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119), 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799)
11:49:38.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:38.210453 1991 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111) [suppressed 5 similar messages]
11:49:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:38.213430 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119): Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111)
11:49:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:38.214020 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799): Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:38.214274 1991 leader_election.cc:302] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dc1e8e3ed3514234ba472ac83b7ea41b; no voters: 4407124148eb4423b5862adc784792b5, 4f82a717bbcd4a0d8a90d4ecdc63f3ae
11:49:38.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:38.214790 2076 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:49:38.448 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:38.449 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:38.451 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:38.452 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:38.452 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:38.453 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:38.453 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:38.453 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:38.456 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:38.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:38.524719 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:38.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:38.524987 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:39.528654 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:39.528949 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:40.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:40.533377 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:40.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:40.533849 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:40.888 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:40.889 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:40.891 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:40.891 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:40.892 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:40.892 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:40.893 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:40.893 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:40.896 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:41.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:41 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869781, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:41.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:41.150683 1983 init.cc:295] Successfully reacquired a new kerberos TGT
11:49:41.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:41.537506 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:41.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:41.537771 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:41.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:41.539534 1970 master_runner.cc:186] Error getting master registration for 127.1.126.62:37119: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111),
11:49:41.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:41.539925 1970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:42.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:42.228336 2086 raft_consensus.cc:482] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:42.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:42.228969 2086 raft_consensus.cc:504] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:42.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:42.230640 2086 leader_election.cc:288] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119), 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799)
11:49:42.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:42.236300 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119): Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111)
11:49:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:42.236882 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799): Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:42.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:42.237146 1991 leader_election.cc:302] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dc1e8e3ed3514234ba472ac83b7ea41b; no voters: 4407124148eb4423b5862adc784792b5, 4f82a717bbcd4a0d8a90d4ecdc63f3ae
11:49:42.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:42.237722 2086 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:49:42.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:42.540902 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:42.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:42.541205 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:43.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:43.434446 947 debug-util.cc:398] Leaking SignalData structure 0x7b080009cb00 after lost signal to thread 882
11:49:43.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:43.435248 947 debug-util.cc:398] Leaking SignalData structure 0x7b08000b03c0 after lost signal to thread 950
11:49:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:43.543979 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:43.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:43.544296 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:44.008 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:44.010 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:44.011 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:44.011 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:44.012 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:44.012 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:44.012 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:44.013 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:44.017 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:44.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:44.549412 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:44.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:44.549800 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:45.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:45.552779 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:45.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:45.553047 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:46.542754 1970 master_runner.cc:186] Error getting master registration for 127.1.126.62:37119: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111),
11:49:46.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:46.543123 1970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:46.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:46.556134 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:46.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:46.556550 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:47.528 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:47.529 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:47.531 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:47.531 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:47.531 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:47.532 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:47.532 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:47.533 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:47.535 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:47.559623 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:47.559959 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:48.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:48.562891 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:48.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:48.563215 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:48.728 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:48.729 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:48.730 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:48.730 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:48.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:48.734 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:48.734 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:48.735 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:48.737 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:49.566286 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:49.566603 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:50.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:50.570359 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:50.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:50.570760 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 19 11:49:51 dist-test-slave-kqf6 krb5kdc[1540](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781869791, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.126.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
11:49:51.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:51.153512 1983 init.cc:295] Successfully reacquired a new kerberos TGT
11:49:51.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:51.546790 1970 master_runner.cc:186] Error getting master registration for 127.1.126.62:37119: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111),
11:49:51.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:51.547358 1970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:51.573837 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:51.574193 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:51.709 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:51.710 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:51.712 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:51.712 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:51.713 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:51.713 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:51.714 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:51.714 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:51.716 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:52.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:52.577514 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:52.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:52.577818 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:52.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:52.584546 2101 raft_consensus.cc:482] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:52.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:52.585062 2101 raft_consensus.cc:504] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:52.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:52.587137 2101 leader_election.cc:288] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119), 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799)
11:49:52.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:52.590219 1991 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111) [suppressed 3 similar messages]
11:49:52.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:52.594094 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119): Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111)
11:49:52.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:52.594810 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799): Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:52.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:52.595196 1991 leader_election.cc:302] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dc1e8e3ed3514234ba472ac83b7ea41b; no voters: 4407124148eb4423b5862adc784792b5, 4f82a717bbcd4a0d8a90d4ecdc63f3ae
11:49:52.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:52.595965 2101 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:49:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:53.580646 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:53.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:53.580904 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:54.583921 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:54.584231 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:55.068 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:55.070 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:55.071 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:55.071 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:55.072 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:55.073 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:55.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:55.073 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:55.077 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:55.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:55.587440 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:55.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:55.587733 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:55.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:55.856087 2108 raft_consensus.cc:482] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
11:49:55.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:55.856623 2108 raft_consensus.cc:504] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4f82a717bbcd4a0d8a90d4ecdc63f3ae" member_type: VOTER last_known_addr { host: "127.1.126.62" port: 37119 } } peers { permanent_uuid: "4407124148eb4423b5862adc784792b5" member_type: VOTER last_known_addr { host: "127.1.126.61" port: 43799 } } peers { permanent_uuid: "dc1e8e3ed3514234ba472ac83b7ea41b" member_type: VOTER last_known_addr { host: "127.1.126.60" port: 36889 } }
11:49:55.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:55.858389 2108 leader_election.cc:288] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119), 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799)
11:49:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:55.864576 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4f82a717bbcd4a0d8a90d4ecdc63f3ae (127.1.126.62:37119): Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111)
11:49:55.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:55.865162 1991 leader_election.cc:334] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4407124148eb4423b5862adc784792b5 (127.1.126.61:43799): Network error: Client connection negotiation failed: client connection to 127.1.126.61:43799: connect: Connection refused (error 111)
11:49:55.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:55.865501 1991 leader_election.cc:302] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dc1e8e3ed3514234ba472ac83b7ea41b; no voters: 4407124148eb4423b5862adc784792b5, 4f82a717bbcd4a0d8a90d4ecdc63f3ae
11:49:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:55.866158 2108 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
11:49:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:56.550297 1970 master_runner.cc:186] Error getting master registration for 127.1.126.62:37119: Network error: Client connection negotiation failed: client connection to 127.1.126.62:37119: connect: Connection refused (error 111),
11:49:56.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:49:56.550679 1970 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
11:49:56.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:56.590746 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:56.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:56.591143 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:57.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:57.594980 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:57.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:57.595301 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:58.008 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:49:58.009 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:49:58.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:58.010 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:58.010 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:49:58.011 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:58.011 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:58.011 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:49:58.014 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:49:58.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:58.598376 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:58.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:58.598744 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:49:59.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:59.601886 2058 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dc1e8e3ed3514234ba472ac83b7ea41b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
11:49:59.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260619 11:49:59.602186 2058 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
11:50:00.288 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.61:43799(127.1.126.61:43799)] connecting to peer
11:50:00.289 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.126.62:37119(127.1.126.62:37119)] connecting to peer
11:50:00.290 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:50:00.290 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.126.61:43799(127.1.126.61:43799)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:50:00.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.61:43799: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799
11:50:00.291 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:50:00.291 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.62:37119(127.1.126.62:37119)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:50:00.292 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.126.62:37119: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
11:50:00.295 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889; will retry
11:50:00.301 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28989), Traces: [0ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [223ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [224ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [587ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [607ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [618ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [659ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [663ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [678ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [694ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [698ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [759ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [807ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [918ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [921ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1015ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1018ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1025ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1454ms] refreshing cache from master, [1455ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1458ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1460ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1462ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1975ms] refreshing cache from master, [1975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1984ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1986ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1986ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [5175ms] refreshing cache from master, [5175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [5176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [5178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [5179ms] too many traces: truncated at 100 traces, deferred=Deferred@492255681(state=PENDING, result=null, callback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478), errback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478)))
11:50:00.307 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.126.61:43799(127.1.126.61:43799)] disconnecting while in state TERMINATED
11:50:00.309 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.126.60:36889(127.1.126.60:36889)] disconnecting while in state READY
11:50:00.309 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.126.62:37119(127.1.126.62:37119)] disconnecting while in state TERMINATED
11:50:00.316 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:50:00.321 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:50:00.323 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.126.60:36889(127.1.126.60:36889)] handling channelInactive
11:50:00.323 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.126.60:36889(127.1.126.60:36889)] cleaning up while in state READY due to: connection closed
11:50:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:50:00.334448 1528 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 1970
11:50:01.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T11:50:01Z chronyd exiting
11:50:02.590 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
11:50:02.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:50:02.592489 796 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 1078
11:50:02.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:50:02.622524 796 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 1241
11:50:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:50:02.655309 796 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 1386
11:50:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:50:02.683316 796 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 808
11:50:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:50:02.715021 796 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 878
11:50:02.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260619 11:50:02.747102 796 external_mini_cluster.cc:1595] Killing /tmp/dist-test-task0xMXX5/build/tsan/bin/kudu with pid 953
11:50:03.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-19T11:50:03Z chronyd exiting
11:50:04.959 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28989), Traces: [0ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [223ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [224ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [587ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [607ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [618ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [659ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [663ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [678ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [694ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [698ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [759ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [807ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [918ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [921ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1015ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1018ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1025ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1454ms] refreshing cache from master, [1455ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1458ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1460ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1462ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1975ms] refreshing cache from master, [1975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1984ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1986ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1986ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [5175ms] refreshing cache from master, [5175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [5176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [5178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [5179ms] too many traces: truncated at 100 traces, deferred=Deferred@492255681(state=PENDING, result=null, callback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478), errback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.126.62:37119
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28989), Traces: [0ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [223ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [224ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [587ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [607ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [618ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [659ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [663ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [678ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [694ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [698ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [759ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [807ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [918ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [921ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1015ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1018ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1025ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1454ms] refreshing cache from master, [1455ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1458ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1460ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1462ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1975ms] refreshing cache from master, [1975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1984ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1986ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1986ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [5175ms] refreshing cache from master, [5175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [5176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [5178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [5179ms] too many traces: truncated at 100 traces, deferred=Deferred@492255681(state=PENDING, result=null, callback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478), errback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.126.62:37119
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
11:50:05.419 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
11:50:05.419 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28989), Traces: [0ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [223ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [224ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [587ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [607ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [618ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [659ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [663ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [678ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [694ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [698ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [759ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [807ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [918ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [921ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1015ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1018ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1025ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1454ms] refreshing cache from master, [1455ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1458ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1460ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1462ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1975ms] refreshing cache from master, [1975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1984ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1986ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1986ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [5175ms] refreshing cache from master, [5175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [5176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [5178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [5179ms] too many traces: truncated at 100 traces, deferred=Deferred@492255681(state=PENDING, result=null, callback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478), errback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.126.62:37119
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=22, TimeoutTracker(timeout=30000, elapsed=28989), Traces: [0ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [145ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [223ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [224ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [587ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [592ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [602ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [607ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [608ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [618ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [634ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [645ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [646ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [659ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [661ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [663ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [664ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [674ms] refreshing cache from master, [675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [678ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [682ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [685ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [694ms] refreshing cache from master, [695ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [698ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [701ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [704ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [754ms] refreshing cache from master, [755ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [757ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [759ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [763ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [764ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [765ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [794ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [799ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [807ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [808ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [914ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [918ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [921ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [923ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [924ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1015ms] refreshing cache from master, [1015ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1018ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1025ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1454ms] refreshing cache from master, [1455ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1456ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1457ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1458ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1460ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1462ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1462ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1975ms] refreshing cache from master, [1975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [1976ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [1978ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.61:43799: Network error: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799, [1978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [1984ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.62:37119: Network error: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [1986ms] Sub RPC ConnectToMaster: received response from server master-127.1.126.60:36889: OK, [1986ms] delaying RPC due to: Service unavailable: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119, [5175ms] refreshing cache from master, [5175ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.61:43799, [5176ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.62:37119, [5178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.126.60:36889, [5179ms] too many traces: truncated at 100 traces, deferred=Deferred@492255681(state=PENDING, result=null, callback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478), errback=(continuation of Deferred@131642024 after retry RPC after error@266848230) -> (continuation of Deferred@2112140078 after retry RPC after error@1507394658) -> (continuation of Deferred@458665987 after retry RPC after error@717196408) -> (continuation of Deferred@1914316738 after retry RPC after error@1668401143) -> (continuation of Deferred@1300866354 after retry RPC after error@453192887) -> (continuation of Deferred@948186146 after retry RPC after error@1437014714) -> (continuation of Deferred@1434944777 after retry RPC after error@1466133914) -> (continuation of Deferred@1319091816 after retry RPC after error@595595211) -> (continuation of Deferred@1254517119 after retry RPC after error@109429248) -> (continuation of Deferred@895107142 after retry RPC after error@378007409) -> (continuation of Deferred@1878257999 after retry RPC after error@1204730739) -> (continuation of Deferred@544316348 after retry RPC after error@161748132) -> (continuation of Deferred@2041443670 after retry RPC after error@1267391539) -> (continuation of Deferred@1728713349 after retry RPC after error@1319255028) -> (continuation of Deferred@1479766067 after retry RPC after error@1170555970) -> (continuation of Deferred@1928128581 after retry RPC after error@768479351) -> (continuation of Deferred@1849646869 after retry RPC after error@28832196) -> (continuation of Deferred@344756087 after retry RPC after error@1226598457) -> (continuation of Deferred@1573863928 after retry RPC after error@1674097236) -> (continuation of Deferred@323972717 after retry RPC after error@960004509) -> (continuation of Deferred@509597635 after retry RPC after error@950246478)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.126.61:43799,127.1.126.62:37119,127.1.126.60:36889) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.61:43799(127.1.126.61:43799): Connection refused: /127.1.126.61:43799,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.126.62:37119(127.1.126.62:37119): Connection refused: /127.1.126.62:37119
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.126.62:37119
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)