04:10:47.552 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
04:10:47.564 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskc0euaK/test-tmp
04:10:47.567 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
04:10:47.571 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskc0euaK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
04:10:47.781 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
04:10:47.932 [DEBUG - main] (MiniKuduCluster.java:195) Response:
04:10:47.933 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
04:10:47.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:47.936787 6431 mini_oidc.cc:152] Starting JWKS server
04:10:47.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:47.940299 6431 webserver.cc:460] Webserver started at http://127.0.0.1:38141/ using document root <none> and password file <none>
04:10:47.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:47.940793 6431 mini_oidc.cc:189] Starting OIDC Discovery server
04:10:47.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:47.942695 6431 webserver.cc:460] Webserver started at http://127.0.0.1:40269/ using document root <none> and password file <none>
04:10:47.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T04:10:47Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
04:10:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T04:10:47Z Disabled control of system clock
04:10:47.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:47.989533 6431 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/wal
04:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/data
04:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:10:47.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/logs
04:10:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/data/info.pb
04:10:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:10:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:10:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:10:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:10:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:10:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:10:47.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:10:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:10:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.254:39717
04:10:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.254
04:10:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:47.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:47.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:47.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:10:47.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:47.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:10:47.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38141/jwks.json with env {}
04:10:48.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.273947 6443 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:10:48.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.274461 6443 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:10:48.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.275007 6443 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:10:48.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.305402 6443 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:10:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.305649 6443 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:10:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.305824 6443 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:10:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.306107 6443 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:10:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.306427 6443 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:10:48.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.306663 6443 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38141/jwks.json
04:10:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:48.339526 6443 master_runner.cc:386] Master server non-default flags:
04:10:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/data
04:10:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/wal
04:10:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:10:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:10:48.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.254:39717
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38141/jwks.json
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/data/info.pb
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.254
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:10:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/logs
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:10:48.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:10:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:10:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:48.340889 6443 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:10:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:48.342973 6443 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:10:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.360862 6450 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.360829 6449 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:48.362493 6443 server_base.cc:1034] running on GCE node
04:10:49.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:48.360903 6452 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:49.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.530933 6443 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:10:49.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.533569 6443 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:10:49.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.534889 6443 hybrid_clock.cc:648] HybridClock initialized: now 1790309449534854 us; error 66 us; skew 500 ppm
04:10:49.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.535638 6443 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:10:49.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.547756 6443 webserver.cc:460] Webserver started at http://127.6.71.254:34857/ using document root <none> and password file <none>
04:10:49.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.548667 6443 fs_manager.cc:359] Metadata directory not provided
04:10:49.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.548853 6443 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:10:49.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.549207 6443 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:10:49.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.553826 6443 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/data/instance:
04:10:49.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cbb4bc7f3e3457583c5a2bc8550904c"
04:10:49.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:49 on dist-test-slave-qxl5"
04:10:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.554982 6443 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/wal/instance:
04:10:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cbb4bc7f3e3457583c5a2bc8550904c"
04:10:49.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:49 on dist-test-slave-qxl5"
04:10:49.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.562568 6443 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
04:10:49.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.567749 6459 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.568877 6443 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
04:10:49.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.569140 6443 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/wal
04:10:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cbb4bc7f3e3457583c5a2bc8550904c"
04:10:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:49 on dist-test-slave-qxl5"
04:10:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.569419 6443 fs_report.cc:383] FS layout report
04:10:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:10:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/wal
04:10:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/wal
04:10:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/data/data
04:10:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:10:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:10:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:10:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:10:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:10:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:10:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:10:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:10:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:10:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:10:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:10:49.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.610477 6443 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:10:49.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.611845 6443 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:10:49.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.612249 6443 kserver.cc:163] Server-wide thread pool size limit: 3276
04:10:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.677820 6443 rpc_server.cc:307] RPC server started. Bound to: 127.6.71.254:39717
04:10:49.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.677942 6510 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.71.254:39717 every 8 connection(s)
04:10:49.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.680351 6443 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/data/info.pb
04:10:49.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.684648 6431 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 6443
04:10:49.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.685010 6431 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-0/wal/instance
04:10:49.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.685353 6511 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:10:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.688326 6431 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:49.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/wal
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/data
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/logs
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/data/info.pb
04:10:49.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:10:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:10:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.253:40329
04:10:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.253
04:10:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:49.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:10:49.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:49.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:10:49.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38141/jwks.json with env {}
04:10:49.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.702986 6511 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } has no permanent_uuid. Determining permanent_uuid...
04:10:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.731464 6511 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } has no permanent_uuid. Determining permanent_uuid...
04:10:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:49.733688 6461 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.71.253:40329: connect: Connection refused (error 111)
04:10:49.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:49.737838 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.253:40329: Network error: Client connection negotiation failed: client connection to 127.6.71.253:40329: connect: Connection refused (error 111)
04:10:49.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.787370 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } attempt: 1
04:10:49.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:49.791392 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.253:40329: Network error: Client connection negotiation failed: client connection to 127.6.71.253:40329: connect: Connection refused (error 111)
04:10:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.859766 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } attempt: 2
04:10:49.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:49.863384 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.253:40329: Network error: Client connection negotiation failed: client connection to 127.6.71.253:40329: connect: Connection refused (error 111)
04:10:49.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:49.954854 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } attempt: 3
04:10:49.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:49.959415 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.253:40329: Network error: Client connection negotiation failed: client connection to 127.6.71.253:40329: connect: Connection refused (error 111)
04:10:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:49.993016 6513 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:10:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:49.993547 6513 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:10:49.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:49.994136 6513 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:10:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.024909 6513 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:10:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.025185 6513 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:10:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.025384 6513 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:10:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.025558 6513 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:10:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.025738 6513 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:10:50.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.025923 6513 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38141/jwks.json
04:10:50.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:50.060283 6513 master_runner.cc:386] Master server non-default flags:
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/data
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/wal
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:10:50.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.253:40329
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38141/jwks.json
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/data/info.pb
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.253
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:50.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/logs
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:10:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:10:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:10:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:10:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:10:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:10:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:10:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:10:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:50.061499 6513 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:10:50.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:50.063284 6513 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:10:50.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.079463 6522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:50.102926 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } attempt: 4
04:10:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.106422 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.253:40329: Network error: Client connection negotiation failed: client connection to 127.6.71.253:40329: connect: Connection refused (error 111)
04:10:50.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:50.406029 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } attempt: 5
04:10:50.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.410881 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.253:40329: Network error: Client connection negotiation failed: client connection to 127.6.71.253:40329: connect: Connection refused (error 111)
04:10:50.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:50.958444 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } attempt: 6
04:10:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.965929 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.253:40329: Network error: Client connection negotiation failed: client connection to 127.6.71.253:40329: connect: Connection refused (error 111)
04:10:51.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.079512 6525 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:50.079535 6523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:51.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:50.081890 6513 server_base.cc:1034] running on GCE node
04:10:51.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.287443 6513 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:10:51.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.290217 6513 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:10:51.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.291597 6513 hybrid_clock.cc:648] HybridClock initialized: now 1790309451291561 us; error 48 us; skew 500 ppm
04:10:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.292536 6513 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:10:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.299060 6513 webserver.cc:460] Webserver started at http://127.6.71.253:40529/ using document root <none> and password file <none>
04:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.300033 6513 fs_manager.cc:359] Metadata directory not provided
04:10:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.300314 6513 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:10:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.300886 6513 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:10:51.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.305644 6513 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/data/instance:
04:10:51.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca4248dd308d4de6867d6e149cb755bf"
04:10:51.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:51 on dist-test-slave-qxl5"
04:10:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.306851 6513 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/wal/instance:
04:10:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca4248dd308d4de6867d6e149cb755bf"
04:10:51.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:51 on dist-test-slave-qxl5"
04:10:51.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.313865 6513 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
04:10:51.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.319123 6534 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:10:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.320078 6513 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
04:10:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.320467 6513 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/wal
04:10:51.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ca4248dd308d4de6867d6e149cb755bf"
04:10:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:51 on dist-test-slave-qxl5"
04:10:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.320868 6513 fs_report.cc:383] FS layout report
04:10:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:10:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/wal
04:10:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/wal
04:10:51.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/data/data
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:10:51.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:10:51.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.361241 6513 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:10:51.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.362720 6513 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:10:51.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.363243 6513 kserver.cc:163] Server-wide thread pool size limit: 3276
04:10:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.430225 6513 rpc_server.cc:307] RPC server started. Bound to: 127.6.71.253:40329
04:10:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.430320 6585 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.71.253:40329 every 8 connection(s)
04:10:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.432897 6513 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/data/info.pb
04:10:51.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.436691 6431 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 6513
04:10:51.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.437143 6431 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-1/wal/instance
04:10:51.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.438560 6586 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:10:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.440069 6431 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/wal
04:10:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/data
04:10:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:10:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:10:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:10:51.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/logs
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/data/info.pb
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.252:44217
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.252
04:10:51.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:10:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:10:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38141/jwks.json with env {}
04:10:51.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.451449 6586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } has no permanent_uuid. Determining permanent_uuid...
04:10:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.474164 6586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } has no permanent_uuid. Determining permanent_uuid...
04:10:51.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.489992 6586 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } has no permanent_uuid. Determining permanent_uuid...
04:10:51.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.492429 6538 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.498004 6586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:51.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.547492 6586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 1
04:10:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.551160 6586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.619526 6586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 2
04:10:51.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.623219 6586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:51.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.714620 6586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 3
04:10:51.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.718304 6586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.747032 6588 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:10:51.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.747531 6588 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:10:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.748049 6588 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:10:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.781627 6588 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:10:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.781883 6588 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:10:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.782070 6588 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:10:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.782286 6588 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:10:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.782549 6588 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:10:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.782696 6588 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38141/jwks.json
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.816022 6588 master_runner.cc:386] Master server non-default flags:
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/data
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/wal
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.252:44217
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38141/jwks.json
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/data/info.pb
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.252
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:10:51.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/logs
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:10:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:10:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:10:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.817171 6588 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:10:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.819181 6588 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:10:51.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.837905 6598 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:51.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.861755 6586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 4
04:10:51.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.865582 6586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.026362 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } attempt: 7
04:10:52.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.040068 6511 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } has no permanent_uuid. Determining permanent_uuid...
04:10:52.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:52.046870 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:52.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.105314 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 1
04:10:52.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:52.108729 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:52.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.165115 6586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 5
04:10:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:52.168702 6586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:52.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.190066 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 2
04:10:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:52.193575 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:52.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.279219 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 3
04:10:52.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:52.284181 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.424572 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 4
04:10:52.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:52.429090 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:52.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.712532 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 5
04:10:52.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:52.716149 6511 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:52.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:52.716142 6586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 6
04:10:52.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:52.722759 6586 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.71.252:44217: Network error: Client connection negotiation failed: client connection to 127.6.71.252:44217: connect: Connection refused (error 111)
04:10:53.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.837971 6601 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:53.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:51.837991 6599 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:53.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:51.839437 6588 server_base.cc:1034] running on GCE node
04:10:53.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.068216 6588 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:10:53.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.071266 6588 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:10:53.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.072690 6588 hybrid_clock.cc:648] HybridClock initialized: now 1790309453072651 us; error 56 us; skew 500 ppm
04:10:53.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.073475 6588 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:10:53.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.079888 6588 webserver.cc:460] Webserver started at http://127.6.71.252:36711/ using document root <none> and password file <none>
04:10:53.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.080814 6588 fs_manager.cc:359] Metadata directory not provided
04:10:53.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.080987 6588 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:10:53.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.081357 6588 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:10:53.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.086022 6588 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/data/instance:
04:10:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f79d2bf228b4481eb3419bc3d0c9f174"
04:10:53.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:53 on dist-test-slave-qxl5"
04:10:53.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.088557 6588 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/wal/instance:
04:10:53.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f79d2bf228b4481eb3419bc3d0c9f174"
04:10:53.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:53 on dist-test-slave-qxl5"
04:10:53.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.095964 6588 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
04:10:53.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.101485 6613 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:10:53.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.102623 6588 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
04:10:53.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.102973 6588 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/wal
04:10:53.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f79d2bf228b4481eb3419bc3d0c9f174"
04:10:53.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:53 on dist-test-slave-qxl5"
04:10:53.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.103330 6588 fs_report.cc:383] FS layout report
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/wal
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/wal
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/data/data
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:10:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:10:53.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.143651 6588 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:10:53.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.145185 6588 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:10:53.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.145700 6588 kserver.cc:163] Server-wide thread pool size limit: 3276
04:10:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.212203 6588 rpc_server.cc:307] RPC server started. Bound to: 127.6.71.252:44217
04:10:53.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.212289 6664 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.71.252:44217 every 8 connection(s)
04:10:53.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.214869 6588 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/data/info.pb
04:10:53.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.219153 6431 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 6588
04:10:53.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.219573 6431 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/master-2/wal/instance
04:10:53.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.219934 6665 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:10:53.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.238991 6665 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } has no permanent_uuid. Determining permanent_uuid...
04:10:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.262614 6665 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } has no permanent_uuid. Determining permanent_uuid...
04:10:53.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.268536 6511 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 6
04:10:53.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.276466 6665 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } has no permanent_uuid. Determining permanent_uuid...
04:10:53.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.293766 6511 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c: Bootstrap starting.
04:10:53.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.300040 6511 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c: Neither blocks nor log segments found. Creating new log.
04:10:53.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.301757 6511 log.cc:824] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c: Log is configured to *not* fsync() on all Append() calls
04:10:53.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.304914 6665 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: Bootstrap starting.
04:10:53.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.306304 6511 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c: No bootstrap required, opened a new log
04:10:53.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.312546 6665 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: Neither blocks nor log segments found. Creating new log.
04:10:53.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.314633 6665 log.cc:824] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: Log is configured to *not* fsync() on all Append() calls
04:10:53.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.323719 6665 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: No bootstrap required, opened a new log
04:10:53.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.333187 6511 raft_consensus.cc:348] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.334015 6511 raft_consensus.cc:374] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:10:53.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.334317 6511 raft_consensus.cc:729] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7cbb4bc7f3e3457583c5a2bc8550904c, State: Initialized, Role: FOLLOWER
04:10:53.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.335245 6511 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.337960 6677 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:53.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.338837 6677 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: This master's current role is: FOLLOWER
04:10:53.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.339864 6511 sys_catalog.cc:564] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: configured and running, proceeding with master startup.
04:10:53.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.341482 6677 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:10:53.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.343123 6677 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.347837 6677 leader_election.cc:288] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ca4248dd308d4de6867d6e149cb755bf (127.6.71.253:40329), f79d2bf228b4481eb3419bc3d0c9f174 (127.6.71.252:44217)
04:10:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.354198 6665 raft_consensus.cc:348] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.355242 6665 raft_consensus.cc:374] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:10:53.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.355542 6665 raft_consensus.cc:729] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f79d2bf228b4481eb3419bc3d0c9f174, State: Initialized, Role: FOLLOWER
04:10:53.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.356617 6665 consensus_queue.cc:260] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.359485 6685 sys_catalog.cc:455] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.361022 6685 sys_catalog.cc:458] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [sys.catalog]: This master's current role is: FOLLOWER
04:10:53.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.361231 6665 sys_catalog.cc:564] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [sys.catalog]: configured and running, proceeding with master startup.
04:10:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.361462 6640 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" is_pre_election: true
04:10:53.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.362473 6640 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7cbb4bc7f3e3457583c5a2bc8550904c in term 0.
04:10:53.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.364025 6463 leader_election.cc:302] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7cbb4bc7f3e3457583c5a2bc8550904c, f79d2bf228b4481eb3419bc3d0c9f174; no voters:
04:10:53.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.365244 6677 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 0 FOLLOWER]: Leader pre-election won for term 1
04:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.365892 6677 raft_consensus.cc:482] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
04:10:53.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.366581 6677 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 0 FOLLOWER]: Advancing to term 1
04:10:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:53.370497 6465 tablet.cc:2369] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:10:53.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.374112 6677 raft_consensus.cc:504] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.381929 6640 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f79d2bf228b4481eb3419bc3d0c9f174"
04:10:53.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.382476 6640 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [term 0 FOLLOWER]: Advancing to term 1
04:10:53.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.388845 6640 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7cbb4bc7f3e3457583c5a2bc8550904c in term 1.
04:10:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.388288 6677 leader_election.cc:288] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [CANDIDATE]: Term 1 election: Requested vote from peers ca4248dd308d4de6867d6e149cb755bf (127.6.71.253:40329), f79d2bf228b4481eb3419bc3d0c9f174 (127.6.71.252:44217)
04:10:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.390125 6463 leader_election.cc:302] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7cbb4bc7f3e3457583c5a2bc8550904c, f79d2bf228b4481eb3419bc3d0c9f174; no voters:
04:10:53.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.391434 6677 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 1 FOLLOWER]: Leader election won for term 1
04:10:53.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.399736 6677 raft_consensus.cc:686] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [term 1 LEADER]: Becoming Leader. State: Replica: 7cbb4bc7f3e3457583c5a2bc8550904c, State: Running, Role: LEADER
04:10:53.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:53.400540 6619 tablet.cc:2369] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:10:53.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.401993 6677 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:53.410053 6701 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:10:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:53.411011 6701 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:10:53.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.417757 6700 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cbb4bc7f3e3457583c5a2bc8550904c. Latest consensus state: current_term: 1 leader_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:53.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.418622 6700 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: This master's current role is: LEADER
04:10:53.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:53.429566 6697 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:10:53.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:53.436817 6697 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:10:53.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.448776 6705 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
04:10:53.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.472743 6705 catalog_manager.cc:1484] Initializing Kudu cluster ID...
04:10:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.428574 6588 master_runner.cc:186] Error getting master registration for 127.6.71.253:40329: OK, instance_id { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" instance_seqno: 1790309451412076 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:10:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.500413 6588 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:10:53.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.414379 6443 master_runner.cc:186] Error getting master registration for 127.6.71.253:40329: OK, instance_id { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" instance_seqno: 1790309451412076 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:10:53.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.507246 6443 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:10:53.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.516614 6640 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [term 1 FOLLOWER]: Refusing update from remote peer 7cbb4bc7f3e3457583c5a2bc8550904c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
04:10:53.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.523090 6677 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [LEADER]: Connected to new peer: Peer: permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
04:10:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.564996 6685 sys_catalog.cc:455] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cbb4bc7f3e3457583c5a2bc8550904c. Latest consensus state: current_term: 1 leader_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.565873 6685 sys_catalog.cc:458] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [sys.catalog]: This master's current role is: FOLLOWER
04:10:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.570607 6677 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.571293 6677 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: This master's current role is: LEADER
04:10:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.572444 6705 catalog_manager.cc:1347] Generated new cluster ID: f10b921bd2c6431abf70287b32bd4617
04:10:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.572742 6705 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
04:10:53.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.594995 6707 mvcc.cc:205] Tried to move back new op lower bound from 7333107521586114560 to 7333107521141710848. Current Snapshot: MvccSnapshot[applied={T|T < 7333107521586114560}]
04:10:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.614153 6685 sys_catalog.cc:455] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:53.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.614902 6685 sys_catalog.cc:458] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174 [sys.catalog]: This master's current role is: FOLLOWER
04:10:53.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.643234 6705 catalog_manager.cc:1370] Generated new certificate authority record
04:10:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.647353 6705 catalog_manager.cc:1504] Loading token signing keys...
04:10:53.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.674034 6705 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c: Generated new TSK 0
04:10:53.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.675086 6705 catalog_manager.cc:1514] Initializing in-progress tserver states...
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.704684 6431 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/wal
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/data
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/logs
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/data/info.pb
04:10:53.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.193:0
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.71.193
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.193
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:10:53.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.783342 6586 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } attempt: 7
04:10:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.837451 6586 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf: Bootstrap starting.
04:10:53.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.847573 6586 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf: Neither blocks nor log segments found. Creating new log.
04:10:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.863904 6586 log.cc:824] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf: Log is configured to *not* fsync() on all Append() calls
04:10:53.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.871773 6586 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf: No bootstrap required, opened a new log
04:10:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.931257 6586 raft_consensus.cc:348] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.933691 6586 raft_consensus.cc:374] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:10:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.934511 6586 raft_consensus.cc:729] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ca4248dd308d4de6867d6e149cb755bf, State: Initialized, Role: FOLLOWER
04:10:53.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.937012 6586 consensus_queue.cc:260] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } }
04:10:53.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.950805 6714 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.955755 6714 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [sys.catalog]: This master's current role is: FOLLOWER
04:10:53.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.957617 6559 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [term 0 FOLLOWER]: Advancing to term 1
04:10:53.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.961802 6560 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ca4248dd308d4de6867d6e149cb755bf"
04:10:53.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.963349 6561 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ca4248dd308d4de6867d6e149cb755bf" is_pre_election: true
04:10:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.969286 6559 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [term 1 FOLLOWER]: Refusing update from remote peer 7cbb4bc7f3e3457583c5a2bc8550904c: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
04:10:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.970914 6586 sys_catalog.cc:564] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [sys.catalog]: configured and running, proceeding with master startup.
04:10:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:53.970909 6700 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
04:10:54.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.027690 6714 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cbb4bc7f3e3457583c5a2bc8550904c. Latest consensus state: current_term: 1 leader_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:54.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.028533 6714 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [sys.catalog]: This master's current role is: FOLLOWER
04:10:54.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.029755 6719 sys_catalog.cc:455] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:54.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.036962 6719 sys_catalog.cc:458] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf [sys.catalog]: This master's current role is: FOLLOWER
04:10:54.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.077462 6730 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
04:10:54.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:54.254894 6711 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:10:54.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:54.255823 6711 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:10:54.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:54.256824 6711 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:10:54.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:54.311856 6711 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:10:54.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:54.313233 6711 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.71.193
04:10:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.373140 6711 tablet_server_runner.cc:78] Tablet server non-default flags:
04:10:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/data
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/wal
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.193:0
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/data/info.pb
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.193
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:10:54.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.71.193
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/logs
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:10:54.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.374677 6711 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:10:54.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.376760 6711 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:10:54.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:54.401546 6736 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:54.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.417742 6701 catalog_manager.cc:1259] Loaded cluster ID: f10b921bd2c6431abf70287b32bd4617
04:10:54.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.418303 6701 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: loading cluster ID for follower catalog manager: success
04:10:54.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.428054 6701 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: acquiring CA information for follower catalog manager: success
04:10:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.447615 6701 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f79d2bf228b4481eb3419bc3d0c9f174: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
04:10:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.586550 6700 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7cbb4bc7f3e3457583c5a2bc8550904c" member_type: VOTER last_known_addr { host: "127.6.71.254" port: 39717 } } peers { permanent_uuid: "ca4248dd308d4de6867d6e149cb755bf" member_type: VOTER last_known_addr { host: "127.6.71.253" port: 40329 } } peers { permanent_uuid: "f79d2bf228b4481eb3419bc3d0c9f174" member_type: VOTER last_known_addr { host: "127.6.71.252" port: 44217 } } }
04:10:54.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:54.588042 6700 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cbb4bc7f3e3457583c5a2bc8550904c [sys.catalog]: This master's current role is: LEADER
04:10:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:55.108572 6730 catalog_manager.cc:1259] Loaded cluster ID: f10b921bd2c6431abf70287b32bd4617
04:10:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:55.108840 6730 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf: loading cluster ID for follower catalog manager: success
04:10:55.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:55.128558 6730 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf: acquiring CA information for follower catalog manager: success
04:10:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:55.161924 6730 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ca4248dd308d4de6867d6e149cb755bf: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
04:10:57.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:54.403594 6737 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:57.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:55.798412 6735 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 6711
04:10:57.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.044835 6738 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2642 milliseconds
04:10:57.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.045313 6711 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.644s user 0.459s sys 0.758s
04:10:57.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.047144 6740 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:57.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.049279 6711 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.649s user 0.459s sys 0.758s
04:10:57.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.049562 6711 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:10:57.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.053231 6711 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:10:57.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.064584 6711 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:10:57.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.072284 6711 hybrid_clock.cc:648] HybridClock initialized: now 1790309457072232 us; error 53 us; skew 500 ppm
04:10:57.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.073418 6711 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:10:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.088214 6711 webserver.cc:460] Webserver started at http://127.6.71.193:45991/ using document root <none> and password file <none>
04:10:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.089277 6711 fs_manager.cc:359] Metadata directory not provided
04:10:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.089510 6711 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:10:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.090148 6711 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:10:57.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.096086 6711 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/data/instance:
04:10:57.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8892b92738ed48f2afc38ace0a72349e"
04:10:57.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:57 on dist-test-slave-qxl5"
04:10:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.097288 6711 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/wal/instance:
04:10:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8892b92738ed48f2afc38ace0a72349e"
04:10:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:57 on dist-test-slave-qxl5"
04:10:57.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.104535 6711 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
04:10:57.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.111317 6750 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:10:57.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.112501 6711 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.112824 6711 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/wal
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8892b92738ed48f2afc38ace0a72349e"
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:57 on dist-test-slave-qxl5"
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.113149 6711 fs_report.cc:383] FS layout report
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/wal
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/wal
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/data/data
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:10:57.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:10:57.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:10:57.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.157454 6711 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:10:57.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.158917 6711 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:10:57.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.159395 6711 kserver.cc:163] Server-wide thread pool size limit: 3276
04:10:57.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.163532 6711 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:10:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.168954 6711 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:10:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.169174 6711 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:10:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.169482 6711 ts_tablet_manager.cc:610] Registered 0 tablets
04:10:57.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.169672 6711 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:10:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.471894 6711 rpc_server.cc:307] RPC server started. Bound to: 127.6.71.193:39361
04:10:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.471967 6862 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.71.193:39361 every 8 connection(s)
04:10:57.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.476022 6711 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/data/info.pb
04:10:57.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.480155 6431 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 6711
04:10:57.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.480579 6431 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-0/wal/instance
04:10:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.490346 6431 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/wal
04:10:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/data
04:10:57.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:10:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:10:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:10:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:57.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/logs
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/data/info.pb
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.194:0
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.71.194
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.194
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:57.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:10:57.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.571275 6863 heartbeater.cc:347] Connected to a master server at 127.6.71.252:44217
04:10:57.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.571801 6863 heartbeater.cc:464] Registering TS with master...
04:10:57.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.573035 6863 heartbeater.cc:511] Master 127.6.71.252:44217 requested a full tablet report, sending...
04:10:57.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.575525 6865 heartbeater.cc:347] Connected to a master server at 127.6.71.254:39717
04:10:57.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.575871 6865 heartbeater.cc:464] Registering TS with master...
04:10:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.577137 6630 ts_manager.cc:194] Registered new tserver with Master: 8892b92738ed48f2afc38ace0a72349e (127.6.71.193:39361)
04:10:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.577201 6865 heartbeater.cc:511] Master 127.6.71.254:39717 requested a full tablet report, sending...
04:10:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.578281 6864 heartbeater.cc:347] Connected to a master server at 127.6.71.253:40329
04:10:57.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.578629 6864 heartbeater.cc:464] Registering TS with master...
04:10:57.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.579478 6864 heartbeater.cc:511] Master 127.6.71.253:40329 requested a full tablet report, sending...
04:10:57.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.581414 6475 ts_manager.cc:194] Registered new tserver with Master: 8892b92738ed48f2afc38ace0a72349e (127.6.71.193:39361)
04:10:57.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.583187 6550 ts_manager.cc:194] Registered new tserver with Master: 8892b92738ed48f2afc38ace0a72349e (127.6.71.193:39361)
04:10:57.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.584082 6475 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.71.193:60467
04:10:57.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.894898 6870 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:10:57.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.895434 6870 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:10:57.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.896004 6870 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:10:57.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.927240 6870 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:10:57.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.928243 6870 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.71.194
04:10:57.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.962406 6870 tablet_server_runner.cc:78] Tablet server non-default flags:
04:10:57.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/data
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/wal
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.194:0
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/data/info.pb
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.194
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.71.194
04:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/logs
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.963893 6870 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:10:57.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.965906 6870 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:10:57.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.986676 6881 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:58.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:58.588388 6865 heartbeater.cc:503] Master 127.6.71.254:39717 was elected leader, sending a full tablet report...
04:10:59.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.986940 6880 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:59.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:57.988152 6870 server_base.cc:1034] running on GCE node
04:10:59.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:57.988054 6883 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:10:59.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.197679 6870 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:10:59.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.200327 6870 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:10:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.201747 6870 hybrid_clock.cc:648] HybridClock initialized: now 1790309459201703 us; error 62 us; skew 500 ppm
04:10:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.202473 6870 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:10:59.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.208972 6870 webserver.cc:460] Webserver started at http://127.6.71.194:40639/ using document root <none> and password file <none>
04:10:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.209798 6870 fs_manager.cc:359] Metadata directory not provided
04:10:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.210094 6870 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:10:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.210618 6870 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:10:59.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.217355 6870 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/data/instance:
04:10:59.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4fe478e6e7141d88f7636aeda0981d5"
04:10:59.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:59 on dist-test-slave-qxl5"
04:10:59.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.218734 6870 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/wal/instance:
04:10:59.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4fe478e6e7141d88f7636aeda0981d5"
04:10:59.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:59 on dist-test-slave-qxl5"
04:10:59.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.227595 6870 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
04:10:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.233433 6891 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:10:59.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.234475 6870 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.234833 6870 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/wal
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4fe478e6e7141d88f7636aeda0981d5"
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:10:59 on dist-test-slave-qxl5"
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.235176 6870 fs_report.cc:383] FS layout report
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/wal
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/wal
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/data/data
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:10:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:10:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:10:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:10:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:10:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:10:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:10:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:10:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:10:59.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:10:59.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.292578 6870 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:10:59.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.294057 6870 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:10:59.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.294548 6870 kserver.cc:163] Server-wide thread pool size limit: 3276
04:10:59.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.297333 6870 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:10:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.301694 6870 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:10:59.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.301900 6870 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:10:59.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.302222 6870 ts_tablet_manager.cc:610] Registered 0 tablets
04:10:59.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.302426 6870 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:10:59.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.452431 6870 rpc_server.cc:307] RPC server started. Bound to: 127.6.71.194:46601
04:10:59.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.452569 7003 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.71.194:46601 every 8 connection(s)
04:10:59.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.455260 6870 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/data/info.pb
04:10:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.461016 6431 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 6870
04:10:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.461786 6431 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-1/wal/instance
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.477917 6431 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/wal
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/data
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/logs
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/data/info.pb
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.195:0
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.71.195
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.195
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:59.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:59.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:59.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
04:10:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.502476 7005 heartbeater.cc:347] Connected to a master server at 127.6.71.253:40329
04:10:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.503264 7005 heartbeater.cc:464] Registering TS with master...
04:10:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.504565 7005 heartbeater.cc:511] Master 127.6.71.253:40329 requested a full tablet report, sending...
04:10:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.507710 7010 heartbeater.cc:347] Connected to a master server at 127.6.71.254:39717
04:10:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.508024 7010 heartbeater.cc:464] Registering TS with master...
04:10:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.508458 7004 heartbeater.cc:347] Connected to a master server at 127.6.71.252:44217
04:10:59.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.508440 6550 ts_manager.cc:194] Registered new tserver with Master: e4fe478e6e7141d88f7636aeda0981d5 (127.6.71.194:46601)
04:10:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.508814 7004 heartbeater.cc:464] Registering TS with master...
04:10:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.508841 7010 heartbeater.cc:511] Master 127.6.71.254:39717 requested a full tablet report, sending...
04:10:59.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.509644 7004 heartbeater.cc:511] Master 127.6.71.252:44217 requested a full tablet report, sending...
04:10:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.511011 6475 ts_manager.cc:194] Registered new tserver with Master: e4fe478e6e7141d88f7636aeda0981d5 (127.6.71.194:46601)
04:10:59.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.511976 6630 ts_manager.cc:194] Registered new tserver with Master: e4fe478e6e7141d88f7636aeda0981d5 (127.6.71.194:46601)
04:10:59.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.513820 6475 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.71.194:56275
04:10:59.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:59.806905 7014 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:10:59.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:59.807333 7014 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:10:59.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:59.807852 7014 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:10:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:59.838774 7014 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:10:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:59.839525 7014 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.71.195
04:10:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.872527 7014 tablet_server_runner.cc:78] Tablet server non-default flags:
04:10:59.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.71.212:36487
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/data
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/wal
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.71.195:0
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/data/info.pb
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.71.195
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.6.71.254:39717,127.6.71.253:40329,127.6.71.252:44217
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.6.71.195
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/logs
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:10:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:10:59.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.873823 7014 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:10:59.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.875590 7014 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:10:59.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:59.894907 7020 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:00.517856 7010 heartbeater.cc:503] Master 127.6.71.254:39717 was elected leader, sending a full tablet report...
04:11:01.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:59.894917 7023 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:10:59.897332 7014 server_base.cc:1034] running on GCE node
04:11:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:10:59.895253 7021 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.082834 7014 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:11:01.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.085570 7014 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.086997 7014 hybrid_clock.cc:648] HybridClock initialized: now 1790309461086975 us; error 50 us; skew 500 ppm
04:11:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.087738 7014 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:11:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.094883 7014 webserver.cc:460] Webserver started at http://127.6.71.195:38669/ using document root <none> and password file <none>
04:11:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.095726 7014 fs_manager.cc:359] Metadata directory not provided
04:11:01.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.095903 7014 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:11:01.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.096338 7014 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:11:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.101027 7014 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/data/instance:
04:11:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8aa71d652f3c4f0f90493ead397ca918"
04:11:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:01 on dist-test-slave-qxl5"
04:11:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.102005 7014 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/wal/instance:
04:11:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8aa71d652f3c4f0f90493ead397ca918"
04:11:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:01 on dist-test-slave-qxl5"
04:11:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.109094 7014 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
04:11:01.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.114493 7031 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:11:01.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.115556 7014 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.115830 7014 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/wal
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8aa71d652f3c4f0f90493ead397ca918"
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:01 on dist-test-slave-qxl5"
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.116403 7014 fs_report.cc:383] FS layout report
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/wal
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/wal
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/data/data
04:11:01.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:11:01.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:11:01.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.153410 7014 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:11:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.154960 7014 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:11:01.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.155382 7014 kserver.cc:163] Server-wide thread pool size limit: 3276
04:11:01.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.158003 7014 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
04:11:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.162442 7014 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
04:11:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.162664 7014 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
04:11:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.162928 7014 ts_tablet_manager.cc:610] Registered 0 tablets
04:11:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.163122 7014 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
04:11:01.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.329690 7014 rpc_server.cc:307] RPC server started. Bound to: 127.6.71.195:32887
04:11:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.329788 7143 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.71.195:32887 every 8 connection(s)
04:11:01.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.332525 7014 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/data/info.pb
04:11:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.335969 6431 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 7014
04:11:01.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.336468 6431 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster4103483847012445025/ts-2/wal/instance
04:11:01.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.385016 7145 heartbeater.cc:347] Connected to a master server at 127.6.71.253:40329
04:11:01.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.385489 7145 heartbeater.cc:464] Registering TS with master...
04:11:01.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.386605 7145 heartbeater.cc:511] Master 127.6.71.253:40329 requested a full tablet report, sending...
04:11:01.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.387113 7144 heartbeater.cc:347] Connected to a master server at 127.6.71.252:44217
04:11:01.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.387446 7144 heartbeater.cc:464] Registering TS with master...
04:11:01.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.388345 7144 heartbeater.cc:511] Master 127.6.71.252:44217 requested a full tablet report, sending...
04:11:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.389832 7146 heartbeater.cc:347] Connected to a master server at 127.6.71.254:39717
04:11:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.390185 7146 heartbeater.cc:464] Registering TS with master...
04:11:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.390132 6550 ts_manager.cc:194] Registered new tserver with Master: 8aa71d652f3c4f0f90493ead397ca918 (127.6.71.195:32887)
04:11:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.390908 7146 heartbeater.cc:511] Master 127.6.71.254:39717 requested a full tablet report, sending...
04:11:01.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.391047 6630 ts_manager.cc:194] Registered new tserver with Master: 8aa71d652f3c4f0f90493ead397ca918 (127.6.71.195:32887)
04:11:01.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.392993 6475 ts_manager.cc:194] Registered new tserver with Master: 8aa71d652f3c4f0f90493ead397ca918 (127.6.71.195:32887)
04:11:01.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.394588 6475 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.71.195:41095
04:11:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:01.402432 6431 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
04:11:01.413 [DEBUG - main] (MiniKuduCluster.java:195) Response:
04:11:01.414 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
04:11:01.427 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.71.254"
port: 39717
}
bound_http_address {
host: "127.6.71.254"
port: 34857
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.71.253"
port: 40329
}
bound_http_address {
host: "127.6.71.253"
port: 40529
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.71.252"
port: 44217
}
bound_http_address {
host: "127.6.71.252"
port: 36711
}
}
}
04:11:01.433 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
04:11:01.436 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.6.71.193"
port: 39361
}
bound_http_address {
host: "127.6.71.193"
port: 45991
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.6.71.194"
port: 46601
}
bound_http_address {
host: "127.6.71.194"
port: 40639
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.6.71.195"
port: 32887
}
bound_http_address {
host: "127.6.71.195"
port: 38669
}
}
}
04:11:01.437 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
04:11:01.634 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
04:11:01.749 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskc0euaK/test-tmp
04:11:01.750 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
04:11:01.750 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskc0euaK/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
04:11:01.754 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
04:11:02.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:02.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
04:11:02.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:02.079026 7156 mini_oidc.cc:152] Starting JWKS server
04:11:02.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:02.082417 7156 webserver.cc:460] Webserver started at http://127.0.0.1:41665/ using document root <none> and password file <none>
04:11:02.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:02.082883 7156 mini_oidc.cc:189] Starting OIDC Discovery server
04:11:02.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:02.084645 7156 webserver.cc:460] Webserver started at http://127.0.0.1:45317/ using document root <none> and password file <none>
04:11:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
04:11:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/principal' for realm 'KRBTEST.COM',
04:11:02.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
04:11:02.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:02 dist-test-slave-qxl5 krb5kdc[7168](info): setting up network...
04:11:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
04:11:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:02 dist-test-slave-qxl5 krb5kdc[7168](info): set up 2 sockets
04:11:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:02 dist-test-slave-qxl5 krb5kdc[7168](info): commencing operation
04:11:02.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
04:11:02.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:02.399557 7146 heartbeater.cc:503] Master 127.6.71.254:39717 was elected leader, sending a full tablet report...
04:11:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.206939 7156 mini_kdc.cc:121] Time spent starting KDC: real 1.098s user 0.003s sys 0.014s
04:11:03.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
04:11:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:11:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
04:11:03.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
04:11:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:11:03.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
04:11:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
04:11:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:11:03.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
04:11:03.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
04:11:03.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/test-user.keytab.
04:11:03.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/test-user.keytab.
04:11:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:03 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309463, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:03.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:11:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T04:11:03Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
04:11:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T04:11:03Z Disabled control of system clock
04:11:03.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.253.62@KRBTEST.COM; defaulting to no policy
04:11:03.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:11:03.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.253.62@KRBTEST.COM" created.
04:11:03.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:11:03.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.253.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.62.keytab.
04:11:03.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.253.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.62.keytab.
04:11:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:03.392458 7156 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data
04:11:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:11:03.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/logs
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/info.pb
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.62:40733
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.62
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.62.keytab
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.62
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:03.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:11:03.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.701269 7181 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:11:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.701896 7181 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:11:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.702602 7181 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:11:03.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.732530 7181 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:11:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.732784 7181 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:11:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.733227 7181 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:11:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.733593 7181 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:11:03.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.733882 7181 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:11:03.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.734172 7181 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:11:03.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.734386 7181 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41665/jwks.json
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:03.773109 7181 master_runner.cc:386] Master server non-default flags:
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.62.keytab
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.62
04:11:03.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.62:40733
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/info.pb
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.62
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/logs
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:11:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:11:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:11:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:03.774539 7181 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:11:03.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:03.775678 7181 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:11:03.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.793222 7190 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:05.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.793440 7187 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:03.795847 7181 server_base.cc:1034] running on GCE node
04:11:05.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:03.793684 7188 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:05.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.025877 7181 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:11:05.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.028967 7181 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:11:05.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.030602 7181 hybrid_clock.cc:648] HybridClock initialized: now 1790309465030556 us; error 67 us; skew 500 ppm
04:11:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.031468 7181 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:11:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:05 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309465, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.038296 7181 init.cc:377] Logged in from keytab as kudu/127.6.253.62@KRBTEST.COM (short username kudu)
04:11:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.043974 7181 webserver.cc:460] Webserver started at http://127.6.253.62:33053/ using document root <none> and password file <none>
04:11:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.044870 7181 fs_manager.cc:359] Metadata directory not provided
04:11:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.045219 7181 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:11:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.045704 7181 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:11:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.050661 7181 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/instance:
04:11:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "216af31f76984a15b315437df0fb08c1"
04:11:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:05 on dist-test-slave-qxl5"
04:11:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.051827 7181 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal/instance:
04:11:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "216af31f76984a15b315437df0fb08c1"
04:11:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:05 on dist-test-slave-qxl5"
04:11:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.059312 7181 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
04:11:05.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.064890 7199 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:11:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.065968 7181 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
04:11:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.066231 7181 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "216af31f76984a15b315437df0fb08c1"
04:11:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:05 on dist-test-slave-qxl5"
04:11:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.066504 7181 fs_report.cc:383] FS layout report
04:11:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/data
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:11:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:11:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.101692 7181 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:11:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.107316 7181 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:11:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.107709 7181 kserver.cc:163] Server-wide thread pool size limit: 3276
04:11:05.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.183629 7181 rpc_server.cc:307] RPC server started. Bound to: 127.6.253.62:40733
04:11:05.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.183712 7250 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.253.62:40733 every 8 connection(s)
04:11:05.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.186215 7181 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/info.pb
04:11:05.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.191617 7251 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:11:05.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.192708 7156 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 7181
04:11:05.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.193164 7156 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal/instance
04:11:05.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.202741 7251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } has no permanent_uuid. Determining permanent_uuid...
04:11:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.253.61@KRBTEST.COM; defaulting to no policy
04:11:05.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:11:05.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.253.61@KRBTEST.COM" created.
04:11:05.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.224678 7140 debug-util.cc:398] Leaking SignalData structure 0x7b08000b67e0 after lost signal to thread 7015
04:11:05.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.226132 7140 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd80 after lost signal to thread 7143
04:11:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:05 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309465, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.62@KRBTEST.COM for kudu/127.6.253.62@KRBTEST.COM
04:11:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:11:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.253.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.61.keytab.
04:11:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.253.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.61.keytab.
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.247728 7156 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/logs
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/info.pb
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:11:05.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.61:34733
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.61
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.61.keytab
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.61
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:05.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:11:05.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.253492 7251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } has no permanent_uuid. Determining permanent_uuid...
04:11:05.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.255981 7201 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:05.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.264401 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.61:34733: Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:05.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.313910 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } attempt: 1
04:11:05.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.318356 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.61:34733: Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:05.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.386761 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } attempt: 2
04:11:05.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.392369 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.61:34733: Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.484406 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } attempt: 3
04:11:05.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.489171 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.61:34733: Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.586092 7257 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:11:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.586637 7257 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:11:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.587325 7257 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:11:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.621054 7257 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:11:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.621330 7257 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:11:05.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.621644 7257 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:11:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.621842 7257 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:11:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.622105 7257 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:11:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.622342 7257 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:11:05.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.622536 7257 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41665/jwks.json
04:11:05.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.632560 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } attempt: 4
04:11:05.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.639158 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.61:34733: Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.662276 7257 master_runner.cc:386] Master server non-default flags:
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:11:05.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.61.keytab
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.61
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.61:34733
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/info.pb
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.61
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/logs
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:11:05.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:11:05.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:11:05.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:11:05.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:11:05.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.663719 7257 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:11:05.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.664846 7257 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:11:05.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.686846 7266 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:05.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:05.938570 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } attempt: 5
04:11:05.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.947319 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.61:34733: Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:06.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:06.494865 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } attempt: 6
04:11:06.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:06.535169 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.61:34733: Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.689112 7265 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:07.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:05.689698 7268 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.269871 7267 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1582 milliseconds
04:11:07.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.269987 7257 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
04:11:07.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.271858 7257 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:11:07.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.276942 7257 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:11:07.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.278362 7257 hybrid_clock.cc:648] HybridClock initialized: now 1790309467278334 us; error 36 us; skew 500 ppm
04:11:07.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.279197 7257 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:11:07.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:07 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309467, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.289222 7257 init.cc:377] Logged in from keytab as kudu/127.6.253.61@KRBTEST.COM (short username kudu)
04:11:07.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.295691 7257 webserver.cc:460] Webserver started at http://127.6.253.61:42057/ using document root <none> and password file <none>
04:11:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.296635 7257 fs_manager.cc:359] Metadata directory not provided
04:11:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.296866 7257 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:11:07.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.297353 7257 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:11:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.301947 7257 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/instance:
04:11:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62374b528ec54ccf908a9855575faac8"
04:11:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:07 on dist-test-slave-qxl5"
04:11:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.303054 7257 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal/instance:
04:11:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62374b528ec54ccf908a9855575faac8"
04:11:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:07 on dist-test-slave-qxl5"
04:11:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.310848 7257 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.004s
04:11:07.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.316891 7278 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:11:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.317939 7257 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
04:11:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.318209 7257 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62374b528ec54ccf908a9855575faac8"
04:11:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:07 on dist-test-slave-qxl5"
04:11:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.318482 7257 fs_report.cc:383] FS layout report
04:11:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:11:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/data
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:11:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:11:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.359848 7257 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:11:07.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.365371 7257 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:11:07.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.365796 7257 kserver.cc:163] Server-wide thread pool size limit: 3276
04:11:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.500521 7257 rpc_server.cc:307] RPC server started. Bound to: 127.6.253.61:34733
04:11:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.500658 7329 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.253.61:34733 every 8 connection(s)
04:11:07.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.503162 7257 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/info.pb
04:11:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.503993 7156 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 7257
04:11:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.504478 7156 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal/instance
04:11:07.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.510811 7330 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:11:07.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.525790 7330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } has no permanent_uuid. Determining permanent_uuid...
04:11:07.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.6.253.60@KRBTEST.COM; defaulting to no policy
04:11:07.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:11:07.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.6.253.60@KRBTEST.COM" created.
04:11:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:07 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309467, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.61@KRBTEST.COM for kudu/127.6.253.62@KRBTEST.COM
04:11:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
04:11:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.253.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.60.keytab.
04:11:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.6.253.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.60.keytab.
04:11:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.564507 7156 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/logs
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/info.pb
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:11:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.60:38961
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.60
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.60.keytab
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.60
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:07.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:07.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:11:07.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.575954 7330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } has no permanent_uuid. Determining permanent_uuid...
04:11:07.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:07 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309467, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.61@KRBTEST.COM for kudu/127.6.253.61@KRBTEST.COM
04:11:07.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.595592 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } attempt: 7
04:11:07.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.611562 7330 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } has no permanent_uuid. Determining permanent_uuid...
04:11:07.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.614481 7281 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.619069 7330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:07 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309465, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.62@KRBTEST.COM for kudu/127.6.253.61@KRBTEST.COM
04:11:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.633657 7251 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } has no permanent_uuid. Determining permanent_uuid...
04:11:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.639144 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.668560 7330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 1
04:11:07.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.672914 7330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.697536 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 1
04:11:07.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.700960 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.741302 7330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 2
04:11:07.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.744982 7330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.782325 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 2
04:11:07.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.786098 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.836321 7330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 3
04:11:07.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.840163 7330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.871712 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 3
04:11:07.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.876721 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.914549 7336 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:11:07.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.915067 7336 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:11:07.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.915875 7336 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:11:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.947877 7336 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:11:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.948088 7336 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:11:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.948421 7336 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:11:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.948618 7336 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:11:07.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.948812 7336 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:11:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.948993 7336 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:11:07.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.949131 7336 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41665/jwks.json
04:11:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.983608 7330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 4
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.985140 7336 master_runner.cc:386] Master server non-default flags:
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.60.keytab
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.60
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.60:38961
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/info.pb
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.60
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/logs
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:11:07.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:11:07.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:07.987142 7330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:07.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.987223 7336 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:11:07.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:07.988236 7336 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:11:08.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:08.004383 7352 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:08.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:08.017146 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 4
04:11:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:08.020918 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:08.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:08.286576 7330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 5
04:11:08.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:08.290587 7330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:08.304260 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 5
04:11:08.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:08.310173 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:08.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:08.838174 7330 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 6
04:11:08.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:08.846328 7330 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:08.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:08.862591 7251 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } attempt: 6
04:11:08.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:08.870393 7251 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.6.253.60:38961: Network error: Client connection negotiation failed: client connection to 127.6.253.60:38961: connect: Connection refused (error 111)
04:11:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:08.004390 7350 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:08.006018 7336 server_base.cc:1034] running on GCE node
04:11:09.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:08.005124 7349 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:09.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.253352 7336 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:11:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.256187 7336 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:11:09.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.257565 7336 hybrid_clock.cc:648] HybridClock initialized: now 1790309469257535 us; error 39 us; skew 500 ppm
04:11:09.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.258452 7336 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:11:09.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309469, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.265281 7336 init.cc:377] Logged in from keytab as kudu/127.6.253.60@KRBTEST.COM (short username kudu)
04:11:09.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.270841 7336 webserver.cc:460] Webserver started at http://127.6.253.60:33905/ using document root <none> and password file <none>
04:11:09.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.272013 7336 fs_manager.cc:359] Metadata directory not provided
04:11:09.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.272312 7336 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:11:09.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.272779 7336 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
04:11:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.277801 7336 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/instance:
04:11:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acd16fe142dc425b9b703667818b6138"
04:11:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:09 on dist-test-slave-qxl5"
04:11:09.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.278836 7336 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal/instance:
04:11:09.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acd16fe142dc425b9b703667818b6138"
04:11:09.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:09 on dist-test-slave-qxl5"
04:11:09.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.286233 7336 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.002s sys 0.007s
04:11:09.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.292239 7365 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:11:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.293190 7336 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
04:11:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.293532 7336 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acd16fe142dc425b9b703667818b6138"
04:11:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:09 on dist-test-slave-qxl5"
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.293886 7336 fs_report.cc:383] FS layout report
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/data
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:11:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:11:09.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.335798 7336 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:11:09.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.341511 7336 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:11:09.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.342036 7336 kserver.cc:163] Server-wide thread pool size limit: 3276
04:11:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.420701 7336 rpc_server.cc:307] RPC server started. Bound to: 127.6.253.60:38961
04:11:09.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.420763 7416 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.253.60:38961 every 8 connection(s)
04:11:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.423950 7336 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/info.pb
04:11:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.425933 7156 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 7336
04:11:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.426424 7156 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal/instance
04:11:09.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.431635 7417 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
04:11:09.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.448370 7417 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } has no permanent_uuid. Determining permanent_uuid...
04:11:09.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309463, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.253.60@KRBTEST.COM
04:11:09.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309469, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.60@KRBTEST.COM for kudu/127.6.253.62@KRBTEST.COM
04:11:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.499118 7417 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } has no permanent_uuid. Determining permanent_uuid...
04:11:09.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309469, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.60@KRBTEST.COM for kudu/127.6.253.61@KRBTEST.COM
04:11:09.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309463, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.253.61@KRBTEST.COM
04:11:09.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.536321 7417 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } has no permanent_uuid. Determining permanent_uuid...
04:11:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309469, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.60@KRBTEST.COM for kudu/127.6.253.60@KRBTEST.COM
04:11:09.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309463, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.253.62@KRBTEST.COM
04:11:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.567360 7156 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
04:11:09.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.567713 7417 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Bootstrap starting.
04:11:09.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.574882 7417 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Neither blocks nor log segments found. Creating new log.
04:11:09.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.576947 7417 log.cc:824] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Log is configured to *not* fsync() on all Append() calls
04:11:09.578 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:09.579 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
04:11:09.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.582587 7417 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: No bootstrap required, opened a new log
04:11:09.592 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
04:11:09.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
04:11:09.597 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.6.253.62"
port: 40733
}
bound_http_address {
host: "127.6.253.62"
port: 33053
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.6.253.61"
port: 34733
}
bound_http_address {
host: "127.6.253.61"
port: 42057
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.6.253.60"
port: 38961
}
bound_http_address {
host: "127.6.253.60"
port: 33905
}
}
}
04:11:09.597 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
04:11:09.597 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:09.597 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
04:11:09.600 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
04:11:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.611637 7417 raft_consensus.cc:348] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.612390 7417 raft_consensus.cc:374] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:11:09.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.612682 7417 raft_consensus.cc:729] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: acd16fe142dc425b9b703667818b6138, State: Initialized, Role: FOLLOWER
04:11:09.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.613598 7417 consensus_queue.cc:260] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:09.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.616313 7427 sys_catalog.cc:455] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } } }
04:11:09.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.617487 7427 sys_catalog.cc:458] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [sys.catalog]: This master's current role is: FOLLOWER
04:11:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.618571 7417 sys_catalog.cc:564] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [sys.catalog]: configured and running, proceeding with master startup.
04:11:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309469, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:09.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:11:09.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:09.648186 7439 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:09.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:09.648496 7439 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:09.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.644552 7336 master_runner.cc:186] Error getting master registration for 127.6.253.62:40733: OK, instance_id { permanent_uuid: "216af31f76984a15b315437df0fb08c1" instance_seqno: 1790309465163757 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
04:11:09.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.680680 7336 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:11:09.694 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:09.697 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc
04:11:09.705 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:11:09.707 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc
04:11:09.708 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:11:09.713 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
04:11:09.715 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
04:11:09.715 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
04:11:09.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.739485 7427 raft_consensus.cc:482] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:11:09.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.740075 7427 raft_consensus.cc:504] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:09.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.744567 7427 leader_election.cc:288] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733), 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733)
04:11:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:09 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309469, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:09.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
04:11:09.799 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:09.799 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc
04:11:09.802 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
04:11:09.802 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.253.62:40733
04:11:09.804 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
04:11:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.805469 7156 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 7181
04:11:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:09.833712 7368 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.6.253.62:40733 (error 108)
04:11:09.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:09.838532 7368 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733): Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111)
04:11:09.843 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:09.843 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.253.61:34733
04:11:09.843 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
04:11:09.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.843873 7156 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 7257
04:11:09.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:09.847641 7371 tablet.cc:2369] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:11:09.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:09.869 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.6.253.60:38961
04:11:09.869 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
04:11:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:09.869791 7367 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733): Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:09.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.870177 7156 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 7336
04:11:09.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:09.895 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.253.62:40733
04:11:09.898 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
04:11:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:09.899308 7156 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data
04:11:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:11:09.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/logs
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/info.pb
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.62:40733
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.62
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33053
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.62.keytab
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.62
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:11:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.215124 7445 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:11:10.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.215677 7445 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:11:10.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.216351 7445 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:11:10.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.248677 7445 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:11:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.248884 7445 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:11:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.249136 7445 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:11:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.249323 7445 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:11:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.249497 7445 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:11:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.249679 7445 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:11:10.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.249809 7445 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41665/jwks.json
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:10.285794 7445 master_runner.cc:386] Master server non-default flags:
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.62.keytab
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.62
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.62:40733
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/info.pb
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.62
04:11:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33053
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/logs
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:11:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:10.286913 7445 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:11:10.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:10.287842 7445 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:11:10.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.304901 7452 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:11.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.305929 7455 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:11.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:10.306561 7445 server_base.cc:1034] running on GCE node
04:11:11.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:10.304915 7453 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:11.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.554499 7445 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:11:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.557742 7445 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:11:11.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.559216 7445 hybrid_clock.cc:648] HybridClock initialized: now 1790309471559175 us; error 36 us; skew 500 ppm
04:11:11.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.560410 7445 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:11:11.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:11 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309471, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.566473 7445 init.cc:377] Logged in from keytab as kudu/127.6.253.62@KRBTEST.COM (short username kudu)
04:11:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.572384 7445 webserver.cc:460] Webserver started at http://127.6.253.62:33053/ using document root <none> and password file <none>
04:11:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.573421 7445 fs_manager.cc:359] Metadata directory not provided
04:11:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.573676 7445 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:11:11.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.581686 7445 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
04:11:11.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.585965 7463 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:11:11.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.587549 7445 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
04:11:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.587893 7445 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "216af31f76984a15b315437df0fb08c1"
04:11:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:05 on dist-test-slave-qxl5"
04:11:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.590284 7445 fs_report.cc:383] FS layout report
04:11:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:11:11.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/data
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:11:11.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:11:11.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.627192 7445 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:11:11.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.633644 7445 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:11:11.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.634037 7445 kserver.cc:163] Server-wide thread pool size limit: 3276
04:11:11.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.713529 7445 rpc_server.cc:307] RPC server started. Bound to: 127.6.253.62:40733
04:11:11.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.713658 7514 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.253.62:40733 every 8 connection(s)
04:11:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.716325 7445 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/data/info.pb
04:11:11.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.720242 7156 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 7445
04:11:11.721 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:11.721 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.253.61:34733
04:11:11.721 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.722844 7156 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/logs
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/info.pb
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:11:11.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:11:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:11.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.61:34733
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.61
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42057
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.61.keytab
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.61
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:11.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:11:11.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.728824 7515 sys_catalog.cc:263] Verifying existing consensus state
04:11:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260925 04:11:11.729825 7515 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:11:11.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.730280 7445 master.cc:537] Master@127.6.253.62:40733 shutting down...
04:11:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:11.800519 7445 master.cc:552] Master@127.6.253.62:40733 shutdown complete.
04:11:11.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:11:12.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.065712 7517 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:11:12.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.066321 7517 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:11:12.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.066906 7517 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:11:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.098722 7517 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:11:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.098953 7517 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:11:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.099213 7517 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:11:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.099408 7517 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:11:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.099586 7517 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:11:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.099759 7517 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:11:12.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.099900 7517 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41665/jwks.json
04:11:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:12.138486 7517 master_runner.cc:386] Master server non-default flags:
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.61.keytab
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.61
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.61:34733
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:11:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/info.pb
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.61
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42057
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/logs
04:11:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:12.139927 7517 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:11:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:12.141155 7517 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:11:12.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.159793 7525 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:13.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.160702 7526 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:12.163825 7517 server_base.cc:1034] running on GCE node
04:11:13.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:12.159812 7528 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:13.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.413395 7517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:11:13.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.416460 7517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:11:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.417891 7517 hybrid_clock.cc:648] HybridClock initialized: now 1790309473417860 us; error 42 us; skew 500 ppm
04:11:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.418664 7517 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:11:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:13 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309473, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:13.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.431496 7517 init.cc:377] Logged in from keytab as kudu/127.6.253.61@KRBTEST.COM (short username kudu)
04:11:13.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.438166 7517 webserver.cc:460] Webserver started at http://127.6.253.61:42057/ using document root <none> and password file <none>
04:11:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.439342 7517 fs_manager.cc:359] Metadata directory not provided
04:11:13.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.439600 7517 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:11:13.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.449707 7517 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.004s sys 0.004s
04:11:13.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.455179 7536 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:11:13.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.456451 7517 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.456815 7517 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "62374b528ec54ccf908a9855575faac8"
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:07 on dist-test-slave-qxl5"
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.459983 7517 fs_report.cc:383] FS layout report
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/data
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:11:13.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:11:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:11:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:11:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:11:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:11:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:11:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:11:13.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:11:13.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.530890 7517 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:11:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.536662 7517 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:11:13.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.537182 7517 kserver.cc:163] Server-wide thread pool size limit: 3276
04:11:13.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.609540 7517 rpc_server.cc:307] RPC server started. Bound to: 127.6.253.61:34733
04:11:13.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.609658 7587 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.253.61:34733 every 8 connection(s)
04:11:13.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.612057 7517 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/data/info.pb
04:11:13.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.619532 7156 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 7517
04:11:13.620 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:13.621 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.6.253.60:38961
04:11:13.621 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
04:11:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.622011 7588 sys_catalog.cc:263] Verifying existing consensus state
04:11:13.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.622637 7156 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/logs
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/info.pb
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.60:38961
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.60
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33905
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.60.keytab
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.60
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:13.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
04:11:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260925 04:11:13.622872 7588 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:11:13.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.623301 7517 master.cc:537] Master@127.6.253.61:34733 shutting down...
04:11:13.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:13.691771 7517 master.cc:552] Master@127.6.253.61:34733 shutdown complete.
04:11:13.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
04:11:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.947989 7590 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
04:11:13.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.948658 7590 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
04:11:13.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.949359 7590 flags.cc:425] Enabled unsafe flag: --never_fsync=true
04:11:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.980298 7590 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
04:11:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.980551 7590 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
04:11:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.980926 7590 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
04:11:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.981235 7590 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
04:11:13.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.981554 7590 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
04:11:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.981859 7590 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
04:11:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:13.982082 7590 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41665/jwks.json
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:14.016848 7590 master_runner.cc:386] Master server non-default flags:
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.6.253.20:36881
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/krb5kdc/kudu_127.6.253.60.keytab
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.6.253.60
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.6.253.60:38961
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41665/jwks.json
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
04:11:14.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/info.pb
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.6.253.60
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33905
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/logs
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 25 Sep 2026 02:53:07 UTC on ccycloud.quasar-ytgwka.root.comops.site
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4671577
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
04:11:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:14.018203 7590 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
04:11:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:14.019291 7590 file_cache.cc:493] Constructed file cache file cache with capacity 419430
04:11:14.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:14.036161 7597 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:15.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:14.036226 7600 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:14.038727 7590 server_base.cc:1034] running on GCE node
04:11:15.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:14.036346 7598 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
04:11:15.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.253772 7590 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
04:11:15.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.256855 7590 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
04:11:15.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.258283 7590 hybrid_clock.cc:648] HybridClock initialized: now 1790309475258252 us; error 51 us; skew 500 ppm
04:11:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.259037 7590 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
04:11:15.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:15 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309475, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:15.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.265084 7590 init.cc:377] Logged in from keytab as kudu/127.6.253.60@KRBTEST.COM (short username kudu)
04:11:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.270341 7590 webserver.cc:460] Webserver started at http://127.6.253.60:33905/ using document root <none> and password file <none>
04:11:15.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.271243 7590 fs_manager.cc:359] Metadata directory not provided
04:11:15.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.271487 7590 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
04:11:15.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.279551 7590 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
04:11:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.283914 7609 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
04:11:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.285053 7590 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
04:11:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.285392 7590 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data,/tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acd16fe142dc425b9b703667818b6138"
04:11:15.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-25 04:11:09 on dist-test-slave-qxl5"
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.287619 7590 fs_report.cc:383] FS layout report
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/wal
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/data
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
04:11:15.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
04:11:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.323175 7590 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
04:11:15.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.328822 7590 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
04:11:15.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.329308 7590 kserver.cc:163] Server-wide thread pool size limit: 3276
04:11:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.400920 7590 rpc_server.cc:307] RPC server started. Bound to: 127.6.253.60:38961
04:11:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.401077 7660 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.253.60:38961 every 8 connection(s)
04:11:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.403579 7590 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskc0euaK/test-tmp/mini-kudu-cluster1296407910983163801/master-2/data/info.pb
04:11:15.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.410444 7156 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu as pid 7590
04:11:15.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
04:11:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.414415 7661 sys_catalog.cc:263] Verifying existing consensus state
04:11:15.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.419404 7661 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Bootstrap starting.
04:11:15.457 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:15.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.458554 7661 log.cc:824] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Log is configured to *not* fsync() on all Append() calls
04:11:15.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.492175 7661 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
04:11:15.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.493072 7661 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Bootstrap complete.
04:11:15.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.527645 7661 raft_consensus.cc:348] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.531425 7661 raft_consensus.cc:374] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
04:11:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.531762 7661 raft_consensus.cc:729] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: acd16fe142dc425b9b703667818b6138, State: Initialized, Role: FOLLOWER
04:11:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.532797 7661 consensus_queue.cc:260] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:15.533 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:15.534 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.6.253.60:38961(127.6.253.60:38961)] connecting to peer
04:11:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.536571 7666 sys_catalog.cc:455] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } } }
04:11:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.537465 7666 sys_catalog.cc:458] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [sys.catalog]: This master's current role is: FOLLOWER
04:11:15.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.540030 7661 sys_catalog.cc:564] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [sys.catalog]: configured and running, proceeding with master startup.
04:11:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:15.585908 7615 tablet.cc:2369] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: Can't schedule compaction. Clean time has not been advanced past its initial value.
04:11:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:15.587352 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:15.587653 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:15.603 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:15.604 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:15.605 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:15.606 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:15.608 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:15.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:15.610 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.6.253.60:38961(127.6.253.60:38961)] Successfully connected to peer
04:11:15.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.581620 7590 master_runner.cc:186] Error getting master registration for 127.6.253.62:40733: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111),
04:11:15.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.615242 7590 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:11:15.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.685456 7666 raft_consensus.cc:482] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:11:15.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.685995 7666 raft_consensus.cc:504] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:15.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:15.690650 7612 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111)
04:11:15.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.692836 7666 leader_election.cc:288] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733), 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733)
04:11:15.701 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
04:11:15.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:15.703020 7612 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733): Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111)
04:11:15.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:15.714780 7611 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733): Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:15.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.715437 7611 leader_election.cc:302] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: acd16fe142dc425b9b703667818b6138; no voters: 216af31f76984a15b315437df0fb08c1, 62374b528ec54ccf908a9855575faac8
04:11:15.716 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.6.253.60
04:11:15.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:15.716403 7666 raft_consensus.cc:2726] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:11:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:15 dist-test-slave-qxl5 krb5kdc[7168](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309469, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.6.253.60@KRBTEST.COM
04:11:15.973 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xed56a49b, L:/127.0.0.1:44148 - R:/127.6.253.60:38961] using SASL/GSSAPI
04:11:15.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:15.997 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:15.998 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.003 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.003 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.003 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.008 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.008 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.009 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:16.015 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:16.017 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.017 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.018 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.019 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.020 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.020 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.021 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.023 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:16.036 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:16.038 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.039 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.040 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.040 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.047 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.047 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.048 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:16.075 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:16.076 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.084 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.084 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.085 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.087 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.087 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.087 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:16.115 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:16.117 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.117 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.118 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.120 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.122 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.122 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.122 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.125 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:16.176 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:16.177 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.178 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.179 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.179 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.181 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.182 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.182 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.184 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:16.195 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:16.197 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.197 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.197 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.198 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.201 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.202 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.202 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.204 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:16.436 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:16.437 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.438 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.440 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.441 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.446 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.447 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.447 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.449 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:16.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:16.591001 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:16.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:16.591316 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:16.895 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:16.897 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.898 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.898 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:16.899 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:16.905 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.906 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.907 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:16.908 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:17.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:17.079460 6507 debug-util.cc:398] Leaking SignalData structure 0x7b0800083300 after lost signal to thread 6444
04:11:17.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:17.080192 6507 debug-util.cc:398] Leaking SignalData structure 0x7b080005c260 after lost signal to thread 6510
04:11:17.456 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:17.456 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:17.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:17.470 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:17.471 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:17.471 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:17.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:17.471 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:17.473 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:17.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:17.553395 7686 raft_consensus.cc:482] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:11:17.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:17.553720 7686 raft_consensus.cc:504] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:17.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:17.555361 7686 leader_election.cc:288] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733), 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733)
04:11:17.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:17.561640 7612 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733): Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111)
04:11:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:17.562959 7611 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733): Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:17.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:17.563280 7611 leader_election.cc:302] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: acd16fe142dc425b9b703667818b6138; no voters: 216af31f76984a15b315437df0fb08c1, 62374b528ec54ccf908a9855575faac8
04:11:17.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:17.563776 7686 raft_consensus.cc:2726] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:11:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:17.594120 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:17.594336 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:18.535 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:18.537 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:18.537 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:18.538 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:18.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:18.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:18.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:18.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:18.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:18.597205 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:18.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:18.597437 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:19.515 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:19.516 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:19.517 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:19.518 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:19.518 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:19.519 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:19.519 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:19.519 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:19.521 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:19.600086 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:19.600380 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:20.145956 7695 raft_consensus.cc:482] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:11:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:20.146339 7695 raft_consensus.cc:504] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:20.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:20.147672 7695 leader_election.cc:288] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733), 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733)
04:11:20.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:20.152088 7612 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733): Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111)
04:11:20.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:20.152511 7611 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733): Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:20.152822 7611 leader_election.cc:302] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: acd16fe142dc425b9b703667818b6138; no voters: 216af31f76984a15b315437df0fb08c1, 62374b528ec54ccf908a9855575faac8
04:11:20.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:20.153427 7695 raft_consensus.cc:2726] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:11:20.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:20.509608 6859 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5e40 after lost signal to thread 6731
04:11:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:20.510355 6859 debug-util.cc:398] Leaking SignalData structure 0x7b08000afd20 after lost signal to thread 6862
04:11:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:20.604558 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:20.604892 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:20.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:20.617336 7590 master_runner.cc:186] Error getting master registration for 127.6.253.62:40733: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111),
04:11:20.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:20.617741 7590 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:11:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:21.609230 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:21.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:21.609568 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:22.612375 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:22.612608 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:22.616 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:22.617 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:22.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:22.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:22.618 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:22.619 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:22.620 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:22.620 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:22.622 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:23.602843 7701 raft_consensus.cc:482] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:11:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:23.603255 7701 raft_consensus.cc:504] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:23.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:23.604810 7701 leader_election.cc:288] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733), 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733)
04:11:23.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:23.606594 7612 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111) [suppressed 5 similar messages]
04:11:23.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:23.609016 7611 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733): Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:23.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:23.609473 7612 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733): Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111)
04:11:23.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:23.609745 7612 leader_election.cc:302] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: acd16fe142dc425b9b703667818b6138; no voters: 216af31f76984a15b315437df0fb08c1, 62374b528ec54ccf908a9855575faac8
04:11:23.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:23.610463 7701 raft_consensus.cc:2726] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:11:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:23.615167 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:23.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:23.615368 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:24.055 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:24.057 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:24.058 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:24.058 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:24.059 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:24.059 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:24.059 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:24.060 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:24.062 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:24.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:24.618939 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:24.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:24.619246 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:25.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:25 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309485, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:25.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:25.269254 7605 init.cc:295] Successfully reacquired a new kerberos TGT
04:11:25.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:25.621284 7590 master_runner.cc:186] Error getting master registration for 127.6.253.62:40733: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111),
04:11:25.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:25.621871 7590 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:11:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:25.622816 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:25.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:25.623040 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:26.625723 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:26.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:26.625962 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:27.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:27.628631 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:27.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:27.628862 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:27.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:27.997 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:27.999 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:27.999 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:27.999 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:27.999 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:27.999 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:27.999 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:28.001 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:28.631775 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:28.632014 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:29.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:29.485800 7712 raft_consensus.cc:482] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:11:29.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:29.486227 7712 raft_consensus.cc:504] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:29.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:29.487902 7712 leader_election.cc:288] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733), 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733)
04:11:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:29.489935 7612 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111) [suppressed 1 similar messages]
04:11:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:29.492969 7611 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733): Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:29.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:29.493423 7612 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733): Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111)
04:11:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:29.493758 7612 leader_election.cc:302] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: acd16fe142dc425b9b703667818b6138; no voters: 216af31f76984a15b315437df0fb08c1, 62374b528ec54ccf908a9855575faac8
04:11:29.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:29.494391 7712 raft_consensus.cc:2726] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:11:29.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:29.634657 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:29.634953 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:29.835 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:29.837 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:29.837 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:29.837 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:29.838 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:29.840 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:29.840 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:29.840 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:29.842 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:30.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:30.624508 7590 master_runner.cc:186] Error getting master registration for 127.6.253.62:40733: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111),
04:11:30.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:30.624888 7590 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:11:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:30.637694 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:30.637919 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:31.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:31.640647 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:31.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:31.640887 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:32.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:32.643788 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:32.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:32.644089 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:33.096 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:33.097 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:33.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:33.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:33.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:33.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:33.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:33.100 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:33.102 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:33.646813 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:33.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:33.647038 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:34.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:34.179281 7722 raft_consensus.cc:482] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
04:11:34.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:34.179913 7722 raft_consensus.cc:504] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "216af31f76984a15b315437df0fb08c1" member_type: VOTER last_known_addr { host: "127.6.253.62" port: 40733 } } peers { permanent_uuid: "62374b528ec54ccf908a9855575faac8" member_type: VOTER last_known_addr { host: "127.6.253.61" port: 34733 } } peers { permanent_uuid: "acd16fe142dc425b9b703667818b6138" member_type: VOTER last_known_addr { host: "127.6.253.60" port: 38961 } }
04:11:34.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:34.182049 7722 leader_election.cc:288] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733), 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733)
04:11:34.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:34.187134 7611 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 62374b528ec54ccf908a9855575faac8 (127.6.253.61:34733): Network error: Client connection negotiation failed: client connection to 127.6.253.61:34733: connect: Connection refused (error 111)
04:11:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:34.187696 7612 leader_election.cc:334] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 216af31f76984a15b315437df0fb08c1 (127.6.253.62:40733): Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111)
04:11:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:34.188009 7612 leader_election.cc:302] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: acd16fe142dc425b9b703667818b6138; no voters: 216af31f76984a15b315437df0fb08c1, 62374b528ec54ccf908a9855575faac8
04:11:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:34.188663 7722 raft_consensus.cc:2726] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
04:11:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:34.650411 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:34.650774 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:35.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 25 04:11:35 dist-test-slave-qxl5 krb5kdc[7168](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790309495, etypes {rep=17 tkt=17 ses=17}, kudu/127.6.253.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
04:11:35.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:35.271484 7605 init.cc:295] Successfully reacquired a new kerberos TGT
04:11:35.295 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:35.297 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:35.298 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:35.298 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:35.298 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:35.299 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:35.299 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:35.299 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:35.302 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:35.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:35.627843 7590 master_runner.cc:186] Error getting master registration for 127.6.253.62:40733: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111),
04:11:35.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:35.628319 7590 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:11:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:35.653728 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:35.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:35.653970 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:36.095 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:36.097 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:36.097 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:36.098 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:36.098 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:36.099 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:36.099 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:36.099 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:36.101 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:36.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:36.656670 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:36.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:36.656894 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:37.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:37.659488 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:37.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:37.659747 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:38.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:38.662629 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:38.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:38.662876 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:39.535 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:39.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:39.536 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:39.537 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:39.537 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:39.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:39.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:39.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:39.540 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:39.665611 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:39.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:39.665844 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:40.630738 7590 master_runner.cc:186] Error getting master registration for 127.6.253.62:40733: Network error: Client connection negotiation failed: client connection to 127.6.253.62:40733: connect: Connection refused (error 111),
04:11:40.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:40.631163 7590 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
04:11:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:40.668431 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:40.668672 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:40.735 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:40.736 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:40.737 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:40.737 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:40.737 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:40.738 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:40.738 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:40.738 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:40.740 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:41.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:41.008299 7000 debug-util.cc:398] Leaking SignalData structure 0x7b08000afba0 after lost signal to thread 6874
04:11:41.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:41.009068 7000 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb080 after lost signal to thread 7003
04:11:41.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:41.671241 7678 catalog_manager.cc:1558] T 00000000000000000000000000000000 P acd16fe142dc425b9b703667818b6138: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
04:11:41.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260925 04:11:41.671521 7678 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
04:11:42.055 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.62:40733(127.6.253.62:40733)] connecting to peer
04:11:42.056 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.6.253.61:34733(127.6.253.61:34733)] connecting to peer
04:11:42.057 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:42.057 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.6.253.62:40733(127.6.253.62:40733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:42.057 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.62:40733: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733
04:11:42.058 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:42.058 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.61:34733(127.6.253.61:34733)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:42.058 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.6.253.61:34733: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
04:11:42.060 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961; will retry
04:11:42.065 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26646), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [188ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [570ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [579ms] refreshing cache from master, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [586ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [589ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [591ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [592ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [598ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [601ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [604ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [605ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [618ms] refreshing cache from master, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [623ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [629ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [631ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [635ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [658ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [667ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [670ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [673ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [674ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [698ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [701ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [705ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [707ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [708ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [762ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [765ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [767ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [778ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [781ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [785ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [787ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1018ms] refreshing cache from master, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1021ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1030ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1031ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1032ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1477ms] refreshing cache from master, [1478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1481ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1488ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1489ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1491ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2038ms] refreshing cache from master, [2038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [2039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [2040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2055ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [2056ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3118ms] refreshing cache from master, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [3120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [3121ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [3123ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3126ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [3126ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [4098ms] refreshing cache from master, [4098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [4099ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [4101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [4101ms] too many traces: truncated at 100 traces, deferred=Deferred@608295333(state=PENDING, result=null, callback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176), errback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176)))
04:11:42.069 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.253.60:38961(127.6.253.60:38961)] disconnecting while in state READY
04:11:42.070 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.253.62:40733(127.6.253.62:40733)] disconnecting while in state TERMINATED
04:11:42.070 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.6.253.61:34733(127.6.253.61:34733)] disconnecting while in state TERMINATED
04:11:42.076 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:11:42.079 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:11:42.080 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.6.253.60:38961(127.6.253.60:38961)] handling channelInactive
04:11:42.080 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.6.253.60:38961(127.6.253.60:38961)] cleaning up while in state READY due to: connection closed
04:11:42.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:42.089783 7156 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 7590
04:11:43.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T04:11:43Z chronyd exiting
04:11:44.311 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
04:11:44.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:44.312194 6431 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 6711
04:11:44.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:44.337785 6431 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 6870
04:11:44.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:44.364662 6431 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 7014
04:11:44.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:44.392803 6431 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 6443
04:11:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:44.423959 6431 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 6513
04:11:44.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260925 04:11:44.454375 6431 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskc0euaK/build/tsan/bin/kudu with pid 6588
04:11:45.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-25T04:11:45Z chronyd exiting
04:11:46.623 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26646), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [188ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [570ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [579ms] refreshing cache from master, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [586ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [589ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [591ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [592ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [598ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [601ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [604ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [605ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [618ms] refreshing cache from master, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [623ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [629ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [631ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [635ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [658ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [667ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [670ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [673ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [674ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [698ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [701ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [705ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [707ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [708ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [762ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [765ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [767ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [778ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [781ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [785ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [787ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1018ms] refreshing cache from master, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1021ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1030ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1031ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1032ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1477ms] refreshing cache from master, [1478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1481ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1488ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1489ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1491ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2038ms] refreshing cache from master, [2038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [2039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [2040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2055ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [2056ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3118ms] refreshing cache from master, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [3120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [3121ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [3123ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3126ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [3126ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [4098ms] refreshing cache from master, [4098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [4099ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [4101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [4101ms] too many traces: truncated at 100 traces, deferred=Deferred@608295333(state=PENDING, result=null, callback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176), errback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.253.61:34733
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26646), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [188ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [570ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [579ms] refreshing cache from master, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [586ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [589ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [591ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [592ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [598ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [601ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [604ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [605ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [618ms] refreshing cache from master, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [623ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [629ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [631ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [635ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [658ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [667ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [670ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [673ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [674ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [698ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [701ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [705ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [707ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [708ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [762ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [765ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [767ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [778ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [781ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [785ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [787ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1018ms] refreshing cache from master, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1021ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1030ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1031ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1032ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1477ms] refreshing cache from master, [1478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1481ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1488ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1489ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1491ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2038ms] refreshing cache from master, [2038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [2039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [2040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2055ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [2056ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3118ms] refreshing cache from master, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [3120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [3121ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [3123ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3126ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [3126ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [4098ms] refreshing cache from master, [4098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [4099ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [4101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [4101ms] too many traces: truncated at 100 traces, deferred=Deferred@608295333(state=PENDING, result=null, callback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176), errback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.253.61:34733
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)
04:11:47.260 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
04:11:47.261 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26646), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [188ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [570ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [579ms] refreshing cache from master, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [586ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [589ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [591ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [592ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [598ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [601ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [604ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [605ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [618ms] refreshing cache from master, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [623ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [629ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [631ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [635ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [658ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [667ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [670ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [673ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [674ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [698ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [701ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [705ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [707ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [708ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [762ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [765ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [767ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [778ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [781ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [785ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [787ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1018ms] refreshing cache from master, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1021ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1030ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1031ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1032ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1477ms] refreshing cache from master, [1478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1481ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1488ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1489ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1491ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2038ms] refreshing cache from master, [2038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [2039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [2040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2055ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [2056ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3118ms] refreshing cache from master, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [3120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [3121ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [3123ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3126ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [3126ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [4098ms] refreshing cache from master, [4098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [4099ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [4101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [4101ms] too many traces: truncated at 100 traces, deferred=Deferred@608295333(state=PENDING, result=null, callback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176), errback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.253.61:34733
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=26646), Traces: [0ms] refreshing cache from master, [30ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [116ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [117ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [188ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [192ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [570ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [575ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [579ms] refreshing cache from master, [580ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [581ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [582ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [586ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [589ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [591ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [592ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [598ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [601ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [601ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [604ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [605ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [606ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [618ms] refreshing cache from master, [619ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [620ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [621ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [623ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [629ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [631ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [635ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [658ms] refreshing cache from master, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [659ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [667ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [670ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [673ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [674ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [698ms] refreshing cache from master, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [700ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [701ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [705ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [707ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [708ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [758ms] refreshing cache from master, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [761ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [762ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [765ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [767ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [778ms] refreshing cache from master, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [780ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [781ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [785ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [787ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [787ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1018ms] refreshing cache from master, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1021ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1030ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1031ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1032ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1477ms] refreshing cache from master, [1478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [1479ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [1481ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [1482ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [1488ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [1489ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [1491ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2038ms] refreshing cache from master, [2038ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [2039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [2040ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [2054ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [2055ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [2056ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3118ms] refreshing cache from master, [3118ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [3119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [3120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [3121ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.62:40733: Network error: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733, [3123ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.61:34733: Network error: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [3126ms] Sub RPC ConnectToMaster: received response from server master-127.6.253.60:38961: OK, [3126ms] delaying RPC due to: Service unavailable: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733, [4098ms] refreshing cache from master, [4098ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.62:40733, [4099ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.61:34733, [4101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.6.253.60:38961, [4101ms] too many traces: truncated at 100 traces, deferred=Deferred@608295333(state=PENDING, result=null, callback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176), errback=(continuation of Deferred@1226759388 after retry RPC after error@1661791437) -> (continuation of Deferred@137349080 after retry RPC after error@571063177) -> (continuation of Deferred@500067112 after retry RPC after error@429954776) -> (continuation of Deferred@1875350323 after retry RPC after error@16623793) -> (continuation of Deferred@1854149805 after retry RPC after error@1930381811) -> (continuation of Deferred@545395370 after retry RPC after error@1305976363) -> (continuation of Deferred@54053440 after retry RPC after error@175196933) -> (continuation of Deferred@987107235 after retry RPC after error@1258416148) -> (continuation of Deferred@237916670 after retry RPC after error@235342429) -> (continuation of Deferred@69642025 after retry RPC after error@1084468977) -> (continuation of Deferred@1023325882 after retry RPC after error@169674394) -> (continuation of Deferred@993257247 after retry RPC after error@1715077806) -> (continuation of Deferred@800596917 after retry RPC after error@1307822350) -> (continuation of Deferred@1776075952 after retry RPC after error@1598391383) -> (continuation of Deferred@831431041 after retry RPC after error@1650894087) -> (continuation of Deferred@1838159039 after retry RPC after error@511984230) -> (continuation of Deferred@1514302110 after retry RPC after error@224113228) -> (continuation of Deferred@1619225268 after retry RPC after error@1546163120) -> (continuation of Deferred@360121393 after retry RPC after error@1164840193) -> (continuation of Deferred@1044584914 after retry RPC after error@644786361) -> (continuation of Deferred@185956041 after retry RPC after error@1042707631) -> (continuation of Deferred@403535664 after retry RPC after error@1265971176)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.6.253.62:40733,127.6.253.61:34733,127.6.253.60:38961) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.62:40733(127.6.253.62:40733): Connection refused: /127.6.253.62:40733,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.6.253.61:34733(127.6.253.61:34733): Connection refused: /127.6.253.61:34733
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.6.253.61:34733
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)