22:22:36.487 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
22:22:36.501 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskmjfZXw/test-tmp
22:22:36.504 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
22:22:36.508 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskmjfZXw/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
22:22:36.766 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
22:22:36.925 [DEBUG - main] (MiniKuduCluster.java:195) Response:
22:22:36.926 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
22:22:36.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:36.930529 3818 mini_oidc.cc:152] Starting JWKS server
22:22:36.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:36.935130 3818 webserver.cc:460] Webserver started at http://127.0.0.1:37915/ using document root <none> and password file <none>
22:22:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:36.935717 3818 mini_oidc.cc:189] Starting OIDC Discovery server
22:22:36.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:36.937722 3818 webserver.cc:460] Webserver started at http://127.0.0.1:46533/ using document root <none> and password file <none>
22:22:36.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T22:22:36Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
22:22:36.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T22:22:36Z Disabled control of system clock
22:22:36.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:36.988138 3818 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:36.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:36.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/wal
22:22:36.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/data
22:22:36.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:36.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:36.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/logs
22:22:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/data/info.pb
22:22:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:22:36.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.190:42831
22:22:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.190
22:22:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:36.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37915/jwks.json with env {}
22:22:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.284600 3830 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.285158 3830 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:37.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.285745 3830 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.316939 3830 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:22:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.317240 3830 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.317435 3830 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:22:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.317611 3830 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:22:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.317826 3830 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:22:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.318022 3830 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37915/jwks.json
22:22:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:37.355522 3830 master_runner.cc:386] Master server non-default flags:
22:22:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:37.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/data
22:22:37.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/wal
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.190:42831
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37915/jwks.json
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:37.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/data/info.pb
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.190
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/logs
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:37.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:22:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:37.356868 3830 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:37.359136 3830 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:37.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.380139 3836 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:38.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.380254 3837 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:38.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:37.381801 3830 server_base.cc:1034] running on GCE node
22:22:38.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:37.382167 3839 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:38.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.516352 3830 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:38.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.520421 3830 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:38.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.522234 3830 hybrid_clock.cc:648] HybridClock initialized: now 1790461358522183 us; error 74 us; skew 500 ppm
22:22:38.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.523242 3830 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:38.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.530984 3830 webserver.cc:460] Webserver started at http://127.3.186.190:33329/ using document root <none> and password file <none>
22:22:38.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.532091 3830 fs_manager.cc:359] Metadata directory not provided
22:22:38.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.532301 3830 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:38.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.532830 3830 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.538370 3830 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/data/instance:
22:22:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06481d1d048245eb9abbc622f2562b41"
22:22:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:38 on dist-test-slave-tfj8"
22:22:38.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.539794 3830 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/wal/instance:
22:22:38.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06481d1d048245eb9abbc622f2562b41"
22:22:38.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:38 on dist-test-slave-tfj8"
22:22:38.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.548511 3830 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
22:22:38.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.555344 3846 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.556751 3830 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
22:22:38.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.557205 3830 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/wal
22:22:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06481d1d048245eb9abbc622f2562b41"
22:22:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:38 on dist-test-slave-tfj8"
22:22:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.557624 3830 fs_report.cc:383] FS layout report
22:22:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/wal
22:22:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/wal
22:22:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/data/data
22:22:38.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:38.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:38.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:38.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:38.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.600289 3830 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.601820 3830 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:38.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.602226 3830 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.678022 3830 rpc_server.cc:307] RPC server started. Bound to: 127.3.186.190:42831
22:22:38.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.678181 3897 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.186.190:42831 every 8 connection(s)
22:22:38.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.681030 3830 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/data/info.pb
22:22:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.684167 3818 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 3830
22:22:38.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.684757 3818 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-0/wal/instance
22:22:38.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.688374 3898 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:22:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.690315 3818 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:38.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/wal
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/data
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/logs
22:22:38.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/data/info.pb
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:38.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.189:33559
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.189
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:38.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:38.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37915/jwks.json with env {}
22:22:38.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.703886 3898 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } has no permanent_uuid. Determining permanent_uuid...
22:22:38.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.736435 3898 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } has no permanent_uuid. Determining permanent_uuid...
22:22:38.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:38.739377 3848 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.186.189:33559: connect: Connection refused (error 111)
22:22:38.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:38.744164 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.189:33559: Network error: Client connection negotiation failed: client connection to 127.3.186.189:33559: connect: Connection refused (error 111)
22:22:38.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.793787 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } attempt: 1
22:22:38.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:38.798520 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.189:33559: Network error: Client connection negotiation failed: client connection to 127.3.186.189:33559: connect: Connection refused (error 111)
22:22:38.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.866967 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } attempt: 2
22:22:38.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:38.871351 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.189:33559: Network error: Client connection negotiation failed: client connection to 127.3.186.189:33559: connect: Connection refused (error 111)
22:22:38.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:38.962893 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } attempt: 3
22:22:38.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:38.966893 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.189:33559: Network error: Client connection negotiation failed: client connection to 127.3.186.189:33559: connect: Connection refused (error 111)
22:22:38.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:38.994872 3900 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:38.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:38.995438 3900 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:38.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:38.995968 3900 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:39.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.028060 3900 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:22:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.028390 3900 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.028587 3900 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:22:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.028765 3900 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:22:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.028940 3900 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:22:39.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.029090 3900 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37915/jwks.json
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:39.065353 3900 master_runner.cc:386] Master server non-default flags:
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/data
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/wal
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:39.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.189:33559
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37915/jwks.json
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/data/info.pb
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.189
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:39.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/logs
22:22:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:39.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:22:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:39.066784 3900 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:39.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:39.069080 3900 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:39.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.088513 3910 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:39.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:39.110316 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } attempt: 4
22:22:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.115953 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.189:33559: Network error: Client connection negotiation failed: client connection to 127.3.186.189:33559: connect: Connection refused (error 111)
22:22:39.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:39.415547 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } attempt: 5
22:22:39.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.421038 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.189:33559: Network error: Client connection negotiation failed: client connection to 127.3.186.189:33559: connect: Connection refused (error 111)
22:22:39.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:39.968616 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } attempt: 6
22:22:39.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.978670 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.189:33559: Network error: Client connection negotiation failed: client connection to 127.3.186.189:33559: connect: Connection refused (error 111)
22:22:40.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.088516 3909 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:40.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:39.088868 3912 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:40.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.228238 3911 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1137 milliseconds
22:22:40.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.228397 3900 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:22:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.230005 3900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.233568 3900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:40.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.235069 3900 hybrid_clock.cc:648] HybridClock initialized: now 1790461360235016 us; error 75 us; skew 500 ppm
22:22:40.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.236409 3900 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:40.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.244539 3900 webserver.cc:460] Webserver started at http://127.3.186.189:33999/ using document root <none> and password file <none>
22:22:40.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.245623 3900 fs_manager.cc:359] Metadata directory not provided
22:22:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.245831 3900 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:40.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.246256 3900 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:40.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.251993 3900 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/data/instance:
22:22:40.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1766f0305eb84faa9c63ef234cc5f71c"
22:22:40.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:40 on dist-test-slave-tfj8"
22:22:40.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.253476 3900 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/wal/instance:
22:22:40.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1766f0305eb84faa9c63ef234cc5f71c"
22:22:40.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:40 on dist-test-slave-tfj8"
22:22:40.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.262269 3900 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
22:22:40.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.269327 3921 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:40.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.270776 3900 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
22:22:40.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.271286 3900 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/wal
22:22:40.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1766f0305eb84faa9c63ef234cc5f71c"
22:22:40.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:40 on dist-test-slave-tfj8"
22:22:40.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.271839 3900 fs_report.cc:383] FS layout report
22:22:40.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:40.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/wal
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/wal
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/data/data
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.313705 3900 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:40.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.315598 3900 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:40.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.316159 3900 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:40.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.402356 3900 rpc_server.cc:307] RPC server started. Bound to: 127.3.186.189:33559
22:22:40.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.402508 3972 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.186.189:33559 every 8 connection(s)
22:22:40.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.405551 3900 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/data/info.pb
22:22:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.408537 3818 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 3900
22:22:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.409034 3818 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-1/wal/instance
22:22:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.412042 3818 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/wal
22:22:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/data
22:22:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:40.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/logs
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/data/info.pb
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.188:43161
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.188
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:40.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:40.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:40.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:40.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:40.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37915/jwks.json with env {}
22:22:40.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.412642 3973 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:22:40.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.425549 3973 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } has no permanent_uuid. Determining permanent_uuid...
22:22:40.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.455749 3973 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } has no permanent_uuid. Determining permanent_uuid...
22:22:40.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.472666 3973 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } has no permanent_uuid. Determining permanent_uuid...
22:22:40.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.475461 3924 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:40.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.480242 3973 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.529779 3973 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 1
22:22:40.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.534058 3973 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:40.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.602499 3973 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 2
22:22:40.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.606714 3973 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.698108 3973 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 3
22:22:40.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.702184 3973 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.759663 3975 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:40.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.760341 3975 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:40.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.760851 3975 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.794880 3975 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:22:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.795217 3975 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.795423 3975 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:22:40.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.795670 3975 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:22:40.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.795878 3975 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:22:40.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.796030 3975 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37915/jwks.json
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.834847 3975 master_runner.cc:386] Master server non-default flags:
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/data
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/wal
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:40.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.188:43161
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37915/jwks.json
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/data/info.pb
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.188
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:40.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/logs
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:40.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:40.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.836395 3975 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:40.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.838749 3975 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:40.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.845597 3973 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 4
22:22:40.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.850841 3973 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:40.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.860637 3986 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:41.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.039265 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } attempt: 7
22:22:41.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.060086 3898 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } has no permanent_uuid. Determining permanent_uuid...
22:22:41.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:41.067025 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:41.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.125545 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 1
22:22:41.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:41.130489 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.150411 3973 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 5
22:22:41.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:41.154410 3973 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:41.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.211905 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 2
22:22:41.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:41.216652 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:41.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.302139 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 3
22:22:41.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:41.307104 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:41.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.447635 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 4
22:22:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:41.452605 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:41.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.702055 3973 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 6
22:22:41.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:41.708874 3973 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:41.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.736126 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 5
22:22:41.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:41.740092 3898 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.3.186.188:43161: Network error: Client connection negotiation failed: client connection to 127.3.186.188:43161: connect: Connection refused (error 111)
22:22:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.860706 3988 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:40.862630 3975 server_base.cc:1034] running on GCE node
22:22:41.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:40.862365 3985 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:41.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.995738 3975 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:41.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:41.998826 3975 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:42.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.000247 3975 hybrid_clock.cc:648] HybridClock initialized: now 1790461362000170 us; error 92 us; skew 500 ppm
22:22:42.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.001112 3975 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:42.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.007932 3975 webserver.cc:460] Webserver started at http://127.3.186.188:41895/ using document root <none> and password file <none>
22:22:42.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.008906 3975 fs_manager.cc:359] Metadata directory not provided
22:22:42.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.009099 3975 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:42.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.009509 3975 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:42.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.016902 3975 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/data/instance:
22:22:42.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29"
22:22:42.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:42 on dist-test-slave-tfj8"
22:22:42.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.018498 3975 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/wal/instance:
22:22:42.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29"
22:22:42.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:42 on dist-test-slave-tfj8"
22:22:42.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.028615 3975 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.005s
22:22:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.036715 4000 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:42.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.038225 3975 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
22:22:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.038691 3975 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/wal
22:22:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29"
22:22:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:42 on dist-test-slave-tfj8"
22:22:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.039160 3975 fs_report.cc:383] FS layout report
22:22:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/wal
22:22:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/wal
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/data/data
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:42.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.085052 3975 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:42.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.086508 3975 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.086926 3975 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:42.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.162760 3975 rpc_server.cc:307] RPC server started. Bound to: 127.3.186.188:43161
22:22:42.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.162883 4051 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.186.188:43161 every 8 connection(s)
22:22:42.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.165676 3975 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/data/info.pb
22:22:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.171868 4052 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:22:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.172091 3818 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 3975
22:22:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.172590 3818 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/master-2/wal/instance
22:22:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.185185 4052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } has no permanent_uuid. Determining permanent_uuid...
22:22:42.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.218384 4052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } has no permanent_uuid. Determining permanent_uuid...
22:22:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.231913 4052 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } has no permanent_uuid. Determining permanent_uuid...
22:22:42.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.254272 4052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29: Bootstrap starting.
22:22:42.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.260741 4052 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29: Neither blocks nor log segments found. Creating new log.
22:22:42.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.262703 4052 log.cc:824] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29: Log is configured to *not* fsync() on all Append() calls
22:22:42.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.267583 4052 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29: No bootstrap required, opened a new log
22:22:42.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.292671 3898 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 6
22:22:42.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.294165 4052 raft_consensus.cc:348] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.295401 4052 raft_consensus.cc:374] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:22:42.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.295933 4052 raft_consensus.cc:729] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e4c2c37bdecc4c528fc8d64f7b7e9f29, State: Initialized, Role: FOLLOWER
22:22:42.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.297277 4052 consensus_queue.cc:260] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.300488 4062 sys_catalog.cc:455] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:42.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.301383 4062 sys_catalog.cc:458] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: This master's current role is: FOLLOWER
22:22:42.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.302407 4052 sys_catalog.cc:564] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: configured and running, proceeding with master startup.
22:22:42.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.317404 3898 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: Bootstrap starting.
22:22:42.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.325721 3898 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: Neither blocks nor log segments found. Creating new log.
22:22:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.329136 3898 log.cc:824] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: Log is configured to *not* fsync() on all Append() calls
22:22:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:42.338721 4073 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:22:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:42.339100 4073 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:22:42.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.341145 3898 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: No bootstrap required, opened a new log
22:22:42.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:42.342588 4006 tablet.cc:2369] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:22:42.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.326931 3975 master_runner.cc:186] Error getting master registration for 127.3.186.190:42831: OK, instance_id { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" instance_seqno: 1790461358656858 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:22:42.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.355845 3975 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:22:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.374076 3898 raft_consensus.cc:348] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.375078 3898 raft_consensus.cc:374] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:22:42.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.375386 3898 raft_consensus.cc:729] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 06481d1d048245eb9abbc622f2562b41, State: Initialized, Role: FOLLOWER
22:22:42.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.376652 3898 consensus_queue.cc:260] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.380066 4076 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:42.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.380807 4076 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [sys.catalog]: This master's current role is: FOLLOWER
22:22:42.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.382347 3898 sys_catalog.cc:564] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [sys.catalog]: configured and running, proceeding with master startup.
22:22:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:42.433727 4087 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:22:42.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:42.434158 4087 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:22:42.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.420423 3830 master_runner.cc:186] Error getting master registration for 127.3.186.189:33559: OK, instance_id { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" instance_seqno: 1790461360379428 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:22:42.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.450951 3830 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:22:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.504335 4062 raft_consensus.cc:482] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:22:42.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.505056 4062 raft_consensus.cc:504] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.509817 3873 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "06481d1d048245eb9abbc622f2562b41" is_pre_election: true
22:22:42.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.510202 4062 leader_election.cc:288] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 06481d1d048245eb9abbc622f2562b41 (127.3.186.190:42831), 1766f0305eb84faa9c63ef234cc5f71c (127.3.186.189:33559)
22:22:42.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.510829 3873 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e4c2c37bdecc4c528fc8d64f7b7e9f29 in term 0.
22:22:42.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.512511 4002 leader_election.cc:302] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06481d1d048245eb9abbc622f2562b41, e4c2c37bdecc4c528fc8d64f7b7e9f29; no voters:
22:22:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.513787 4062 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 0 FOLLOWER]: Leader pre-election won for term 1
22:22:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.514258 4062 raft_consensus.cc:482] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
22:22:42.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.514597 4062 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 0 FOLLOWER]: Advancing to term 1
22:22:42.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.522308 4062 raft_consensus.cc:504] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.537132 4062 leader_election.cc:288] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [CANDIDATE]: Term 1 election: Requested vote from peers 06481d1d048245eb9abbc622f2562b41 (127.3.186.190:42831), 1766f0305eb84faa9c63ef234cc5f71c (127.3.186.189:33559)
22:22:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.538300 3873 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "06481d1d048245eb9abbc622f2562b41"
22:22:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.538938 3873 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [term 0 FOLLOWER]: Advancing to term 1
22:22:42.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.546269 3873 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e4c2c37bdecc4c528fc8d64f7b7e9f29 in term 1.
22:22:42.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.548115 4002 leader_election.cc:302] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06481d1d048245eb9abbc622f2562b41, e4c2c37bdecc4c528fc8d64f7b7e9f29; no voters:
22:22:42.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.550580 4062 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 1 FOLLOWER]: Leader election won for term 1
22:22:42.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.555322 4062 raft_consensus.cc:686] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [term 1 LEADER]: Becoming Leader. State: Replica: e4c2c37bdecc4c528fc8d64f7b7e9f29, State: Running, Role: LEADER
22:22:42.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.556851 4062 consensus_queue.cc:237] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.581341 4089 sys_catalog.cc:455] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e4c2c37bdecc4c528fc8d64f7b7e9f29. Latest consensus state: current_term: 1 leader_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:42.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.583992 4089 sys_catalog.cc:458] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: This master's current role is: LEADER
22:22:42.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.589097 4091 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
22:22:42.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.596828 4091 catalog_manager.cc:1484] Initializing Kudu cluster ID...
22:22:42.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:42.612202 3852 tablet.cc:2369] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:22:42.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.624769 3873 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [term 1 FOLLOWER]: Refusing update from remote peer e4c2c37bdecc4c528fc8d64f7b7e9f29: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:22:42.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.626814 4089 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
22:22:42.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.654906 4076 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e4c2c37bdecc4c528fc8d64f7b7e9f29. Latest consensus state: current_term: 1 leader_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:42.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.656399 4076 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [sys.catalog]: This master's current role is: FOLLOWER
22:22:42.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.662355 4092 mvcc.cc:205] Tried to move back new op lower bound from 7333729741288308736 to 7333729741046349824. Current Snapshot: MvccSnapshot[applied={T|T < 7333729741288308736}]
22:22:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.668363 4076 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:42.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.669162 4076 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41 [sys.catalog]: This master's current role is: FOLLOWER
22:22:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.680478 4089 sys_catalog.cc:455] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:42.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.681267 4089 sys_catalog.cc:458] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: This master's current role is: LEADER
22:22:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.684546 4091 catalog_manager.cc:1347] Generated new cluster ID: d01725ba89c640f18e9e01a1fea99d8a
22:22:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.684866 4091 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
22:22:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.735023 4091 catalog_manager.cc:1370] Generated new certificate authority record
22:22:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.737617 4091 catalog_manager.cc:1504] Loading token signing keys...
22:22:42.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.769419 3973 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } attempt: 7
22:22:42.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.778353 4091 catalog_manager.cc:5868] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29: Generated new TSK 0
22:22:42.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.779588 4091 catalog_manager.cc:1514] Initializing in-progress tserver states...
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.808854 3818 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/wal
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/data
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/logs
22:22:42.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/data/info.pb
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.129:0
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.186.129
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.129
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:42.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:22:42.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.822247 3973 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c: Bootstrap starting.
22:22:42.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.830464 3973 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c: Neither blocks nor log segments found. Creating new log.
22:22:42.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.833204 3973 log.cc:824] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c: Log is configured to *not* fsync() on all Append() calls
22:22:42.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.847174 3973 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c: No bootstrap required, opened a new log
22:22:42.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.925019 3973 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.927615 3973 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:22:42.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.928592 3973 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1766f0305eb84faa9c63ef234cc5f71c, State: Initialized, Role: FOLLOWER
22:22:42.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.931142 3973 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.952297 4100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:42.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.958550 4100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [sys.catalog]: This master's current role is: FOLLOWER
22:22:42.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.968833 3946 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [term 0 FOLLOWER]: Advancing to term 1
22:22:42.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.975921 3948 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1766f0305eb84faa9c63ef234cc5f71c" is_pre_election: true
22:22:42.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.979075 3946 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [term 1 FOLLOWER]: Refusing update from remote peer e4c2c37bdecc4c528fc8d64f7b7e9f29: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
22:22:42.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.980857 4100 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:22:42.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.980933 4089 consensus_queue.cc:1035] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
22:22:42.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.981797 4100 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } }
22:22:42.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.988945 3873 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1766f0305eb84faa9c63ef234cc5f71c" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "06481d1d048245eb9abbc622f2562b41" is_pre_election: true
22:22:42.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.983984 3947 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1766f0305eb84faa9c63ef234cc5f71c"
22:22:42.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.986028 3973 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [sys.catalog]: configured and running, proceeding with master startup.
22:22:43.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:42.998678 4027 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1766f0305eb84faa9c63ef234cc5f71c" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" is_pre_election: true
22:22:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.000835 3924 leader_election.cc:302] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1766f0305eb84faa9c63ef234cc5f71c; no voters: 06481d1d048245eb9abbc622f2562b41, e4c2c37bdecc4c528fc8d64f7b7e9f29
22:22:43.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.007756 4100 leader_election.cc:288] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 06481d1d048245eb9abbc622f2562b41 (127.3.186.190:42831), e4c2c37bdecc4c528fc8d64f7b7e9f29 (127.3.186.188:43161)
22:22:43.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.063192 4106 mvcc.cc:205] Tried to move back new op lower bound from 7333729741288308736 to 7333729741046349824. Current Snapshot: MvccSnapshot[applied={T|T < 7333729741288308736}]
22:22:43.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.097815 4100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [sys.catalog]: SysCatalogTable state changed. Reason: New leader e4c2c37bdecc4c528fc8d64f7b7e9f29. Latest consensus state: current_term: 1 leader_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.098596 4100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [sys.catalog]: This master's current role is: FOLLOWER
22:22:43.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.099133 4102 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
22:22:43.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.102022 4101 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:43.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.102811 4101 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c [sys.catalog]: This master's current role is: FOLLOWER
22:22:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.157351 4117 catalog_manager.cc:1259] Loaded cluster ID: d01725ba89c640f18e9e01a1fea99d8a
22:22:43.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.157763 4117 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c: loading cluster ID for follower catalog manager: success
22:22:43.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.177731 4117 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c: acquiring CA information for follower catalog manager: success
22:22:43.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.186606 4117 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 1766f0305eb84faa9c63ef234cc5f71c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:22:43.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:43.339998 4098 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:43.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:43.341001 4098 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:43.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:43.342278 4098 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:43.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:43.403219 4098 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:43.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:43.405006 4098 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.186.129
22:22:43.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.440784 4087 catalog_manager.cc:1259] Loaded cluster ID: d01725ba89c640f18e9e01a1fea99d8a
22:22:43.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.441682 4087 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: loading cluster ID for follower catalog manager: success
22:22:43.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.450212 4087 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: acquiring CA information for follower catalog manager: success
22:22:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.467029 4098 tablet_server_runner.cc:78] Tablet server non-default flags:
22:22:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/data
22:22:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/wal
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.129:0
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/data/info.pb
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.129
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.186.129
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/logs
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:43.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.474356 4087 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 06481d1d048245eb9abbc622f2562b41: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
22:22:43.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.475366 4098 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.477744 4098 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:43.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:43.524228 4125 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:43.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.576174 4089 sys_catalog.cc:455] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "06481d1d048245eb9abbc622f2562b41" member_type: VOTER last_known_addr { host: "127.3.186.190" port: 42831 } } peers { permanent_uuid: "1766f0305eb84faa9c63ef234cc5f71c" member_type: VOTER last_known_addr { host: "127.3.186.189" port: 33559 } } peers { permanent_uuid: "e4c2c37bdecc4c528fc8d64f7b7e9f29" member_type: VOTER last_known_addr { host: "127.3.186.188" port: 43161 } } }
22:22:43.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:43.578025 4089 sys_catalog.cc:458] T 00000000000000000000000000000000 P e4c2c37bdecc4c528fc8d64f7b7e9f29 [sys.catalog]: This master's current role is: LEADER
22:22:45.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:44.901036 4123 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 4098
22:22:45.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:45.345319 4098 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.828s user 0.662s sys 0.857s
22:22:45.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:45.371770 4098 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.854s user 0.662s sys 0.857s
22:22:45.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.372030 4098 server_base.cc:1034] running on GCE node
22:22:45.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:45.373078 4127 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:45.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:43.540150 4124 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:45.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.380026 4098 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:45.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.388099 4098 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:45.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.394948 4098 hybrid_clock.cc:648] HybridClock initialized: now 1790461365394889 us; error 86 us; skew 500 ppm
22:22:45.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.396809 4098 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:45.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.428035 4098 webserver.cc:460] Webserver started at http://127.3.186.129:36915/ using document root <none> and password file <none>
22:22:45.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.429965 4098 fs_manager.cc:359] Metadata directory not provided
22:22:45.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.430542 4098 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:45.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.436800 4098 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:45.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.444638 4098 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/data/instance:
22:22:45.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e44497a9476c4fe9be8bc576f0a7658a"
22:22:45.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:45 on dist-test-slave-tfj8"
22:22:45.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.446349 4098 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/wal/instance:
22:22:45.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e44497a9476c4fe9be8bc576f0a7658a"
22:22:45.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:45 on dist-test-slave-tfj8"
22:22:45.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.469893 4098 fs_manager.cc:691] Time spent creating directory manager: real 0.023s user 0.023s sys 0.000s
22:22:45.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.484203 4137 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:45.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.485564 4098 fs_manager.cc:725] Time spent opening block manager: real 0.011s user 0.007s sys 0.004s
22:22:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.485971 4098 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/wal
22:22:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e44497a9476c4fe9be8bc576f0a7658a"
22:22:45.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:45 on dist-test-slave-tfj8"
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.486415 4098 fs_report.cc:383] FS layout report
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/wal
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/wal
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/data/data
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:45.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:45.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.558956 4098 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:45.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.560482 4098 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:45.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.560887 4098 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:45.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.563769 4098 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:22:45.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.568457 4098 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:22:45.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.568733 4098 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:22:45.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.569001 4098 ts_tablet_manager.cc:610] Registered 0 tablets
22:22:45.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.569192 4098 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:22:45.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.852430 4098 rpc_server.cc:307] RPC server started. Bound to: 127.3.186.129:37099
22:22:45.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.852533 4249 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.186.129:37099 every 8 connection(s)
22:22:45.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.855306 4098 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/data/info.pb
22:22:45.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.857291 3818 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4098
22:22:45.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.857823 3818 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-0/wal/instance
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.869654 3818 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/wal
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/data
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/logs
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/data/info.pb
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:45.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.130:0
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.186.130
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.130
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:45.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:22:45.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.964133 4251 heartbeater.cc:347] Connected to a master server at 127.3.186.189:33559
22:22:45.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.964643 4251 heartbeater.cc:464] Registering TS with master...
22:22:45.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.965804 4251 heartbeater.cc:511] Master 127.3.186.189:33559 requested a full tablet report, sending...
22:22:45.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.969306 3938 ts_manager.cc:194] Registered new tserver with Master: e44497a9476c4fe9be8bc576f0a7658a (127.3.186.129:37099)
22:22:45.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.970549 4250 heartbeater.cc:347] Connected to a master server at 127.3.186.188:43161
22:22:45.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.970918 4250 heartbeater.cc:464] Registering TS with master...
22:22:45.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.971776 4250 heartbeater.cc:511] Master 127.3.186.188:43161 requested a full tablet report, sending...
22:22:45.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.974299 4017 ts_manager.cc:194] Registered new tserver with Master: e44497a9476c4fe9be8bc576f0a7658a (127.3.186.129:37099)
22:22:45.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.977152 4017 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.186.129:55329
22:22:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.988978 4257 heartbeater.cc:347] Connected to a master server at 127.3.186.190:42831
22:22:45.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.989372 4257 heartbeater.cc:464] Registering TS with master...
22:22:45.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.990097 4257 heartbeater.cc:511] Master 127.3.186.190:42831 requested a full tablet report, sending...
22:22:45.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:45.993036 3863 ts_manager.cc:194] Registered new tserver with Master: e44497a9476c4fe9be8bc576f0a7658a (127.3.186.129:37099)
22:22:46.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:46.234505 4252 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:46.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:46.235026 4252 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:46.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:46.235649 4252 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:46.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:46.267813 4252 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:46.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:46.269026 4252 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.186.130
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:46.304823 4252 tablet_server_runner.cc:78] Tablet server non-default flags:
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/data
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/wal
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.130:0
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/data/info.pb
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.130
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.186.130
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/logs
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:46.306342 4252 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:46.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:46.308111 4252 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:46.327602 4267 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:46.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:46.982578 4250 heartbeater.cc:503] Master 127.3.186.188:43161 was elected leader, sending a full tablet report...
22:22:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:46.327608 4269 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:46.330703 4266 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:47.522820 4268 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1192 milliseconds
22:22:47.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.522974 4252 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:22:47.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.524390 4252 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:47.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.526962 4252 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.528389 4252 hybrid_clock.cc:648] HybridClock initialized: now 1790461367528360 us; error 144 us; skew 500 ppm
22:22:47.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.529294 4252 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:47.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.536334 4252 webserver.cc:460] Webserver started at http://127.3.186.130:39307/ using document root <none> and password file <none>
22:22:47.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.537364 4252 fs_manager.cc:359] Metadata directory not provided
22:22:47.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.537580 4252 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.538115 4252 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.543522 4252 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/data/instance:
22:22:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b34a9dc69944099b305ac2cd9ec0ca0"
22:22:47.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:47 on dist-test-slave-tfj8"
22:22:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.544813 4252 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/wal/instance:
22:22:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b34a9dc69944099b305ac2cd9ec0ca0"
22:22:47.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:47 on dist-test-slave-tfj8"
22:22:47.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.553537 4252 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
22:22:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.560269 4278 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.561685 4252 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
22:22:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.562095 4252 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/wal
22:22:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0b34a9dc69944099b305ac2cd9ec0ca0"
22:22:47.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:47 on dist-test-slave-tfj8"
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.562525 4252 fs_report.cc:383] FS layout report
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/wal
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/wal
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/data/data
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:47.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.612121 4252 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:47.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.614100 4252 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:47.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.614619 4252 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.618003 4252 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:22:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.623754 4252 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:22:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.623999 4252 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:22:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.624294 4252 ts_tablet_manager.cc:610] Registered 0 tablets
22:22:47.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.624516 4252 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:22:47.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.805047 4252 rpc_server.cc:307] RPC server started. Bound to: 127.3.186.130:42315
22:22:47.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.805200 4390 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.186.130:42315 every 8 connection(s)
22:22:47.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.807644 4252 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/data/info.pb
22:22:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.811184 3818 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4252
22:22:47.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.811911 3818 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-1/wal/instance
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.827610 3818 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/wal
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/data
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/logs
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/data/info.pb
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:47.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.131:0
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.186.131
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.131
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:47.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
22:22:47.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.873586 4391 heartbeater.cc:347] Connected to a master server at 127.3.186.188:43161
22:22:47.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.874243 4391 heartbeater.cc:464] Registering TS with master...
22:22:47.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.875734 4391 heartbeater.cc:511] Master 127.3.186.188:43161 requested a full tablet report, sending...
22:22:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.878772 4017 ts_manager.cc:194] Registered new tserver with Master: 0b34a9dc69944099b305ac2cd9ec0ca0 (127.3.186.130:42315)
22:22:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.880999 4017 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.186.130:34027
22:22:47.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.891817 4392 heartbeater.cc:347] Connected to a master server at 127.3.186.189:33559
22:22:47.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.892220 4392 heartbeater.cc:464] Registering TS with master...
22:22:47.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.892977 4392 heartbeater.cc:511] Master 127.3.186.189:33559 requested a full tablet report, sending...
22:22:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.895803 3938 ts_manager.cc:194] Registered new tserver with Master: 0b34a9dc69944099b305ac2cd9ec0ca0 (127.3.186.130:42315)
22:22:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.896277 4393 heartbeater.cc:347] Connected to a master server at 127.3.186.190:42831
22:22:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.896595 4393 heartbeater.cc:464] Registering TS with master...
22:22:47.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.897213 4393 heartbeater.cc:511] Master 127.3.186.190:42831 requested a full tablet report, sending...
22:22:47.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:47.899776 3863 ts_manager.cc:194] Registered new tserver with Master: 0b34a9dc69944099b305ac2cd9ec0ca0 (127.3.186.130:42315)
22:22:48.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:48.178069 4399 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:48.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:48.178620 4399 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:48.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:48.179165 4399 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:48.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:48.211403 4399 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:48.212740 4399 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.3.186.131
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:48.248659 4399 tablet_server_runner.cc:78] Tablet server non-default flags:
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.3.186.148:43061
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/data
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/wal
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.3.186.131:0
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/data/info.pb
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.3.186.131
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.3.186.190:42831,127.3.186.189:33559,127.3.186.188:43161
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.3.186.131
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/logs
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
22:22:48.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:48.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:48.249989 4399 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:48.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:48.251788 4399 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:48.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:48.274726 4408 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:48.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:48.887416 4391 heartbeater.cc:503] Master 127.3.186.188:43161 was elected leader, sending a full tablet report...
22:22:49.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:48.274837 4407 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:49.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:49.405640 4409 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:22:49.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:48.274731 4410 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:49.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.406227 4399 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:22:49.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.410262 4399 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:49.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.412985 4399 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:49.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.414409 4399 hybrid_clock.cc:648] HybridClock initialized: now 1790461369414309 us; error 122 us; skew 500 ppm
22:22:49.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.415166 4399 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:49.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.422052 4399 webserver.cc:460] Webserver started at http://127.3.186.131:33787/ using document root <none> and password file <none>
22:22:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.422900 4399 fs_manager.cc:359] Metadata directory not provided
22:22:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.423079 4399 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:49.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.423465 4399 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.428228 4399 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/data/instance:
22:22:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46e5c42a2cde480fad4ee5e3147cc3c2"
22:22:49.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:49 on dist-test-slave-tfj8"
22:22:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.429589 4399 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/wal/instance:
22:22:49.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46e5c42a2cde480fad4ee5e3147cc3c2"
22:22:49.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:49 on dist-test-slave-tfj8"
22:22:49.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.437301 4399 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
22:22:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.443017 4418 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.444317 4399 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.444705 4399 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/wal
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "46e5c42a2cde480fad4ee5e3147cc3c2"
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:49 on dist-test-slave-tfj8"
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.445071 4399 fs_report.cc:383] FS layout report
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/wal
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/wal
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/data/data
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:49.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:49.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.485596 4399 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.487044 4399 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:49.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.487465 4399 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.490003 4399 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
22:22:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.494361 4399 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
22:22:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.494560 4399 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
22:22:49.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.494786 4399 ts_tablet_manager.cc:610] Registered 0 tablets
22:22:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.494940 4399 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
22:22:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.672176 4399 rpc_server.cc:307] RPC server started. Bound to: 127.3.186.131:33725
22:22:49.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.672356 4530 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.3.186.131:33725 every 8 connection(s)
22:22:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.674742 4399 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/data/info.pb
22:22:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.684757 3818 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4399
22:22:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.685312 3818 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster57539188996938313/ts-2/wal/instance
22:22:49.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.743961 4531 heartbeater.cc:347] Connected to a master server at 127.3.186.188:43161
22:22:49.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.745235 4532 heartbeater.cc:347] Connected to a master server at 127.3.186.189:33559
22:22:49.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.745301 4531 heartbeater.cc:464] Registering TS with master...
22:22:49.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.745630 4532 heartbeater.cc:464] Registering TS with master...
22:22:49.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.746784 4532 heartbeater.cc:511] Master 127.3.186.189:33559 requested a full tablet report, sending...
22:22:49.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.746870 4531 heartbeater.cc:511] Master 127.3.186.188:43161 requested a full tablet report, sending...
22:22:49.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.747017 4533 heartbeater.cc:347] Connected to a master server at 127.3.186.190:42831
22:22:49.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.747445 4533 heartbeater.cc:464] Registering TS with master...
22:22:49.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.748315 4533 heartbeater.cc:511] Master 127.3.186.190:42831 requested a full tablet report, sending...
22:22:49.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.750312 4017 ts_manager.cc:194] Registered new tserver with Master: 46e5c42a2cde480fad4ee5e3147cc3c2 (127.3.186.131:33725)
22:22:49.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.752529 3938 ts_manager.cc:194] Registered new tserver with Master: 46e5c42a2cde480fad4ee5e3147cc3c2 (127.3.186.131:33725)
22:22:49.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.752626 4017 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.3.186.131:59659
22:22:49.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.755069 3862 ts_manager.cc:194] Registered new tserver with Master: 46e5c42a2cde480fad4ee5e3147cc3c2 (127.3.186.131:33725)
22:22:49.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:49.762140 3818 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
22:22:49.773 [DEBUG - main] (MiniKuduCluster.java:195) Response:
22:22:49.774 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
22:22:49.786 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.3.186.190"
port: 42831
}
bound_http_address {
host: "127.3.186.190"
port: 33329
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.3.186.189"
port: 33559
}
bound_http_address {
host: "127.3.186.189"
port: 33999
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.3.186.188"
port: 43161
}
bound_http_address {
host: "127.3.186.188"
port: 41895
}
}
}
22:22:49.794 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
22:22:49.798 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.3.186.129"
port: 37099
}
bound_http_address {
host: "127.3.186.129"
port: 36915
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.3.186.130"
port: 42315
}
bound_http_address {
host: "127.3.186.130"
port: 39307
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.3.186.131"
port: 33725
}
bound_http_address {
host: "127.3.186.131"
port: 33787
}
}
}
22:22:49.799 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
22:22:50.034 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
22:22:50.204 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskmjfZXw/test-tmp
22:22:50.206 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
22:22:50.206 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskmjfZXw/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
22:22:50.210 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
22:22:50.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:50.729 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
22:22:50.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:50.733065 4542 mini_oidc.cc:152] Starting JWKS server
22:22:50.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:50.737505 4542 webserver.cc:460] Webserver started at http://127.0.0.1:40697/ using document root <none> and password file <none>
22:22:50.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:50.738077 4542 mini_oidc.cc:189] Starting OIDC Discovery server
22:22:50.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:50.740342 4542 webserver.cc:460] Webserver started at http://127.0.0.1:46483/ using document root <none> and password file <none>
22:22:50.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:50.757087 4531 heartbeater.cc:503] Master 127.3.186.188:43161 was elected leader, sending a full tablet report...
22:22:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
22:22:50.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/principal' for realm 'KRBTEST.COM',
22:22:50.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
22:22:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:50 dist-test-slave-tfj8 krb5kdc[4555](info): setting up network...
22:22:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
22:22:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:50 dist-test-slave-tfj8 krb5kdc[4555](info): set up 2 sockets
22:22:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:50 dist-test-slave-tfj8 krb5kdc[4555](info): commencing operation
22:22:50.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
22:22:51.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:51.769788 4542 mini_kdc.cc:121] Time spent starting KDC: real 1.003s user 0.007s sys 0.012s
22:22:51.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
22:22:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:22:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
22:22:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
22:22:51.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:22:51.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
22:22:51.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
22:22:51.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:22:51.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
22:22:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
22:22:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/test-user.keytab.
22:22:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/test-user.keytab.
22:22:51.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:51 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461371, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:22:51.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
22:22:51.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T22:22:51Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
22:22:51.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T22:22:51Z Disabled control of system clock
22:22:51.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.111.190@KRBTEST.COM; defaulting to no policy
22:22:51.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:22:51.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.111.190@KRBTEST.COM" created.
22:22:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:22:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.111.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.190.keytab.
22:22:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.111.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.190.keytab.
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:51.977654 4542 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/logs
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/info.pb
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:51.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.190:39623
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.190
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.190.keytab
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.190
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:51.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:22:52.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.324745 4568 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:52.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.325502 4568 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:52.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.326251 4568 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:52.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.359174 4568 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:22:52.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.359515 4568 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:22:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.359982 4568 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.360318 4568 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:22:52.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.360667 4568 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:22:52.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.360963 4568 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:22:52.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.361191 4568 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40697/jwks.json
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:52.401744 4568 master_runner.cc:386] Master server non-default flags:
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.190.keytab
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.190
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.190:39623
22:22:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/info.pb
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.190
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/logs
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:52.403568 4568 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:52.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:52.405324 4568 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:52.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.431175 4576 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:53.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.431597 4578 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:52.432039 4575 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:53.650754 4577 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1217 milliseconds
22:22:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.650943 4568 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:22:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.652406 4568 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.655448 4568 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.656937 4568 hybrid_clock.cc:648] HybridClock initialized: now 1790461373656878 us; error 75 us; skew 500 ppm
22:22:53.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.657893 4568 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:53.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:53 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461373, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:22:53.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.664407 4568 init.cc:377] Logged in from keytab as kudu/127.4.111.190@KRBTEST.COM (short username kudu)
22:22:53.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.670835 4568 webserver.cc:460] Webserver started at http://127.4.111.190:35157/ using document root <none> and password file <none>
22:22:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.671837 4568 fs_manager.cc:359] Metadata directory not provided
22:22:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.672026 4568 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:53.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.672420 4568 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.677891 4568 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/instance:
22:22:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "505d27ebcfa94756b3c137ba3013da8e"
22:22:53.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:53 on dist-test-slave-tfj8"
22:22:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.679652 4568 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal/instance:
22:22:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "505d27ebcfa94756b3c137ba3013da8e"
22:22:53.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:53 on dist-test-slave-tfj8"
22:22:53.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.690161 4568 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.005s sys 0.004s
22:22:53.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.698871 4587 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:53.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.700558 4568 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.005s sys 0.002s
22:22:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.701022 4568 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "505d27ebcfa94756b3c137ba3013da8e"
22:22:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:53 on dist-test-slave-tfj8"
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.701474 4568 fs_report.cc:383] FS layout report
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/data
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:53.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.750898 4568 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.756589 4568 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.757009 4568 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:53.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.853544 4568 rpc_server.cc:307] RPC server started. Bound to: 127.4.111.190:39623
22:22:53.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.853752 4638 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.111.190:39623 every 8 connection(s)
22:22:53.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.856400 4568 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/info.pb
22:22:53.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.864511 4542 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4568
22:22:53.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.864979 4542 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal/instance
22:22:53.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.866015 4639 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:22:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.111.189@KRBTEST.COM; defaulting to no policy
22:22:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.883065 4639 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } has no permanent_uuid. Determining permanent_uuid...
22:22:53.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:22:53.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.111.189@KRBTEST.COM" created.
22:22:53.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:53 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461373, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.190@KRBTEST.COM for kudu/127.4.111.190@KRBTEST.COM
22:22:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:22:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.111.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.189.keytab.
22:22:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.111.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.189.keytab.
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.916860 4542 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/logs
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/info.pb
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.189:32841
22:22:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.189
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.189.keytab
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.189
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:22:53.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.931346 4639 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } has no permanent_uuid. Determining permanent_uuid...
22:22:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:53.934202 4589 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:22:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:53.939858 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.189:32841: Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:22:53.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:53.989359 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } attempt: 1
22:22:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:53.993674 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.189:32841: Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:22:54.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:54.062047 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } attempt: 2
22:22:54.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.066684 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.189:32841: Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:22:54.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:54.158110 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } attempt: 3
22:22:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.162786 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.189:32841: Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:22:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.296594 4645 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.297372 4645 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.298102 4645 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:54.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:54.306216 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } attempt: 4
22:22:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.310668 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.189:32841: Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:22:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.333750 4645 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:22:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.334062 4645 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:22:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.334474 4645 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.334789 4645 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:22:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.335088 4645 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:22:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.335376 4645 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:22:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.335636 4645 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40697/jwks.json
22:22:54.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:54.377333 4645 master_runner.cc:386] Master server non-default flags:
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.189.keytab
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.189
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:54.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.189:32841
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/info.pb
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.189
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/logs
22:22:54.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:54.379182 4645 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:54.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:54.380769 4645 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:54.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.405198 4653 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:54.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:54.610073 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } attempt: 5
22:22:54.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.615244 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.189:32841: Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:22:55.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.162789 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } attempt: 6
22:22:55.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:55.205925 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.189:32841: Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:22:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.407322 4652 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:55.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:54.405488 4655 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:55.627596 4654 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1221 milliseconds
22:22:55.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.627797 4645 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:22:55.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.629215 4645 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.632205 4645 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:55.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.633574 4645 hybrid_clock.cc:648] HybridClock initialized: now 1790461375633521 us; error 71 us; skew 500 ppm
22:22:55.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.634449 4645 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:55 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461375, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:22:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.641425 4645 init.cc:377] Logged in from keytab as kudu/127.4.111.189@KRBTEST.COM (short username kudu)
22:22:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.648695 4645 webserver.cc:460] Webserver started at http://127.4.111.189:37351/ using document root <none> and password file <none>
22:22:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.649871 4645 fs_manager.cc:359] Metadata directory not provided
22:22:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.650125 4645 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.650671 4645 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.657105 4645 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/instance:
22:22:55.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d517c799d6b471abab944a9df73af84"
22:22:55.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:55 on dist-test-slave-tfj8"
22:22:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.658800 4645 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal/instance:
22:22:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d517c799d6b471abab944a9df73af84"
22:22:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:55 on dist-test-slave-tfj8"
22:22:55.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.667382 4645 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
22:22:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.676844 4667 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.678119 4645 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
22:22:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.678493 4645 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:22:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d517c799d6b471abab944a9df73af84"
22:22:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:55 on dist-test-slave-tfj8"
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.678844 4645 fs_report.cc:383] FS layout report
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/data
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:55.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:55.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.729676 4645 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:55.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.735267 4645 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:55.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.735847 4645 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:55.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.833065 4645 rpc_server.cc:307] RPC server started. Bound to: 127.4.111.189:32841
22:22:55.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.833184 4718 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.111.189:32841 every 8 connection(s)
22:22:55.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.835922 4645 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/info.pb
22:22:55.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.838183 4542 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4645
22:22:55.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.838677 4542 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal/instance
22:22:55.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.845512 4719 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:22:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.4.111.188@KRBTEST.COM; defaulting to no policy
22:22:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:22:55.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.4.111.188@KRBTEST.COM" created.
22:22:55.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.868723 4719 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } has no permanent_uuid. Determining permanent_uuid...
22:22:55.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:55 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461375, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.189@KRBTEST.COM for kudu/127.4.111.190@KRBTEST.COM
22:22:55.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.917871 4719 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } has no permanent_uuid. Determining permanent_uuid...
22:22:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
22:22:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.111.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.188.keytab.
22:22:55.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.4.111.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.188.keytab.
22:22:55.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.921271 4542 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:55.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/logs
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/info.pb
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:22:55.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.188:44163
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.188
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.188.keytab
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.188
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:55.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:22:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:55 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461375, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.189@KRBTEST.COM for kudu/127.4.111.189@KRBTEST.COM
22:22:55.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:55.950312 4719 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } has no permanent_uuid. Determining permanent_uuid...
22:22:55.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:55.952734 4670 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:55.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:55.957343 4719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.007055 4719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 1
22:22:56.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.011425 4719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.079881 4719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 2
22:22:56.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.084163 4719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.175585 4719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 3
22:22:56.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.180258 4719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.266531 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } attempt: 7
22:22:56.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:56 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461373, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.190@KRBTEST.COM for kudu/127.4.111.189@KRBTEST.COM
22:22:56.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.294481 4639 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } has no permanent_uuid. Determining permanent_uuid...
22:22:56.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.294605 4726 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:56.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.295410 4726 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:56.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.296267 4726 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:56.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.300228 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.323683 4719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 4
22:22:56.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.327535 4719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.331981 4726 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:22:56.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.332234 4726 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:22:56.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.332502 4726 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:56.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.332703 4726 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:22:56.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.332906 4726 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:22:56.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.333140 4726 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:22:56.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.333319 4726 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40697/jwks.json
22:22:56.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.358726 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 1
22:22:56.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.362679 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.374529 4726 master_runner.cc:386] Master server non-default flags:
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.188.keytab
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.188
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.188:44163
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/info.pb
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.188
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/logs
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:56.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:56.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.376045 4726 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:56.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.377249 4726 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:56.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.396517 4736 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.444195 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 2
22:22:56.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.450186 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.535742 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 3
22:22:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.541399 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.626912 4719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 5
22:22:56.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.632405 4719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.681914 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 4
22:22:56.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.692361 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:56.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:56.976475 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 5
22:22:56.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.982745 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.179921 4719 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 6
22:22:57.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:57.200450 4719 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:57.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.535341 4639 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } attempt: 6
22:22:57.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:57.546134 4639 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.4.111.188:44163: Network error: Client connection negotiation failed: client connection to 127.4.111.188:44163: connect: Connection refused (error 111)
22:22:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.400590 4739 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:56.396603 4737 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:57.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:57.603786 4738 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1204 milliseconds
22:22:57.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.603955 4726 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:22:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.605398 4726 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:57.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.609787 4726 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:57.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.611266 4726 hybrid_clock.cc:648] HybridClock initialized: now 1790461377611209 us; error 82 us; skew 500 ppm
22:22:57.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.612315 4726 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:57.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:57 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461377, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:22:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.619227 4726 init.cc:377] Logged in from keytab as kudu/127.4.111.188@KRBTEST.COM (short username kudu)
22:22:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.626916 4726 webserver.cc:460] Webserver started at http://127.4.111.188:41285/ using document root <none> and password file <none>
22:22:57.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.628118 4726 fs_manager.cc:359] Metadata directory not provided
22:22:57.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.628378 4726 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:57.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.628942 4726 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
22:22:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.634562 4726 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/instance:
22:22:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfbccaf74c56419c807a394e83895b60"
22:22:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:57 on dist-test-slave-tfj8"
22:22:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.635941 4726 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal/instance:
22:22:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfbccaf74c56419c807a394e83895b60"
22:22:57.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:57 on dist-test-slave-tfj8"
22:22:57.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.644971 4726 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
22:22:57.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.651574 4753 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:57.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.652834 4726 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
22:22:57.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.653244 4726 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:22:57.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfbccaf74c56419c807a394e83895b60"
22:22:57.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:57 on dist-test-slave-tfj8"
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.653611 4726 fs_report.cc:383] FS layout report
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/data
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:57.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:57.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.692217 4726 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:57.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.699820 4726 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.700871 4726 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:57.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.803840 4726 rpc_server.cc:307] RPC server started. Bound to: 127.4.111.188:44163
22:22:57.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.803967 4804 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.111.188:44163 every 8 connection(s)
22:22:57.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.806804 4726 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/info.pb
22:22:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.808733 4542 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4726
22:22:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.809238 4542 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal/instance
22:22:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.815555 4805 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
22:22:57.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.842104 4805 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } has no permanent_uuid. Determining permanent_uuid...
22:22:57.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:57 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461371, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.111.188@KRBTEST.COM
22:22:57.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:57 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461377, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.188@KRBTEST.COM for kudu/127.4.111.190@KRBTEST.COM
22:22:57.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.886385 4805 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } has no permanent_uuid. Determining permanent_uuid...
22:22:57.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:57 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461371, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.111.189@KRBTEST.COM
22:22:57.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:57 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461377, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.188@KRBTEST.COM for kudu/127.4.111.189@KRBTEST.COM
22:22:57.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.913928 4805 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } has no permanent_uuid. Determining permanent_uuid...
22:22:57.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:57 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461371, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.111.190@KRBTEST.COM
22:22:57.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:57 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461377, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.188@KRBTEST.COM for kudu/127.4.111.188@KRBTEST.COM
22:22:57.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.944923 4542 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
22:22:57.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.951406 4805 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: Bootstrap starting.
22:22:57.956 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:57.958 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
22:22:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.959964 4805 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: Neither blocks nor log segments found. Creating new log.
22:22:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.962611 4805 log.cc:824] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: Log is configured to *not* fsync() on all Append() calls
22:22:57.969 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
22:22:57.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:57.970335 4805 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: No bootstrap required, opened a new log
22:22:57.970 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
22:22:57.973 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.4.111.190"
port: 39623
}
bound_http_address {
host: "127.4.111.190"
port: 35157
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.4.111.189"
port: 32841
}
bound_http_address {
host: "127.4.111.189"
port: 37351
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.4.111.188"
port: 44163
}
bound_http_address {
host: "127.4.111.188"
port: 41285
}
}
}
22:22:57.974 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
22:22:57.974 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:57.974 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
22:22:57.975 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
22:22:58.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:57 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461377, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:22:58.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
22:22:58.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.004770 4805 raft_consensus.cc:348] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:22:58.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.005551 4805 raft_consensus.cc:374] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:22:58.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.005761 4805 raft_consensus.cc:729] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfbccaf74c56419c807a394e83895b60, State: Initialized, Role: FOLLOWER
22:22:58.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.006572 4805 consensus_queue.cc:260] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:22:58.008 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:58.010 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc
22:22:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.010998 4816 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } } }
22:22:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.012053 4816 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [sys.catalog]: This master's current role is: FOLLOWER
22:22:58.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.012285 4805 sys_catalog.cc:564] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [sys.catalog]: configured and running, proceeding with master startup.
22:22:58.020 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
22:22:58.021 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc
22:22:58.023 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
22:22:58.026 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
22:22:58.028 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
22:22:58.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
22:22:58.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.052964 4828 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:22:58.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.053583 4828 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:22:58.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:58 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461378, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:22:58.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
22:22:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.045187 4726 master_runner.cc:186] Error getting master registration for 127.4.111.190:39623: OK, instance_id { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" instance_seqno: 1790461373826295 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
22:22:58.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.073413 4726 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:22:58.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:58.077 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc
22:22:58.078 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
22:22:58.078 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.111.190:39623
22:22:58.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
22:22:58.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.080664 4542 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 4568
22:22:58.110 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:58.111 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.111.189:32841
22:22:58.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
22:22:58.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.111657 4542 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 4645
22:22:58.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:58.130 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.4.111.188:44163
22:22:58.130 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
22:22:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.130599 4542 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 4726
22:22:58.150 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:58.151 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.111.190:39623
22:22:58.152 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.154117 4542 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/logs
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/info.pb
22:22:58.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.190:39623
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.190
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35157
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.190.keytab
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.190
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:58.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:58.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:22:58.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.521944 4832 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:22:58.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.522920 4832 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:22:58.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.524099 4832 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:22:58.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.559459 4832 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:22:58.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.559787 4832 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:22:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.560062 4832 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:22:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.560271 4832 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:22:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.560467 4832 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:22:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.560671 4832 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:22:58.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.560815 4832 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40697/jwks.json
22:22:58.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.602415 4832 master_runner.cc:386] Master server non-default flags:
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.190.keytab
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.190
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.190:39623
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/info.pb
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.190
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35157
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/logs
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:22:58.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:22:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:22:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:22:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:22:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:22:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.603919 4832 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:22:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:58.605124 4832 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:22:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.625344 4838 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:59.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.625357 4839 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:59.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:58.626031 4841 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:22:59.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:22:59.785964 4840 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:22:59.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.786078 4832 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:22:59.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.790760 4832 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:22:59.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.793764 4832 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:22:59.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.795213 4832 hybrid_clock.cc:648] HybridClock initialized: now 1790461379795099 us; error 86 us; skew 500 ppm
22:22:59.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.796166 4832 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:22:59.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:22:59 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461379, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:22:59.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.804585 4832 init.cc:377] Logged in from keytab as kudu/127.4.111.190@KRBTEST.COM (short username kudu)
22:22:59.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.810395 4832 webserver.cc:460] Webserver started at http://127.4.111.190:35157/ using document root <none> and password file <none>
22:22:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.811412 4832 fs_manager.cc:359] Metadata directory not provided
22:22:59.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.811707 4832 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:22:59.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.821233 4832 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.008s sys 0.000s
22:22:59.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.825924 4850 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:22:59.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.827188 4832 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.827558 4832 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "505d27ebcfa94756b3c137ba3013da8e"
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:53 on dist-test-slave-tfj8"
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.830098 4832 fs_report.cc:383] FS layout report
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/data
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:22:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:22:59.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.870110 4832 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:22:59.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.876677 4832 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:22:59.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.877238 4832 kserver.cc:163] Server-wide thread pool size limit: 3276
22:22:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.957768 4832 rpc_server.cc:307] RPC server started. Bound to: 127.4.111.190:39623
22:22:59.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.958022 4901 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.111.190:39623 every 8 connection(s)
22:22:59.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.961011 4832 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/data/info.pb
22:22:59.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.967429 4542 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4832
22:22:59.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:22:59.968 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.111.189:32841
22:22:59.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
22:22:59.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.969797 4542 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:59.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:22:59.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/logs
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/info.pb
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:22:59.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.189:32841
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.189
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37351
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.189.keytab
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.189
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:22:59.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.972597 4902 sys_catalog.cc:263] Verifying existing consensus state
22:22:59.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 22:22:59.973560 4902 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
22:22:59.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:22:59.974049 4832 master.cc:537] Master@127.4.111.190:39623 shutting down...
22:23:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:00.041116 4832 master.cc:552] Master@127.4.111.190:39623 shutdown complete.
22:23:00.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
22:23:00.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.313777 4904 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:23:00.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.314585 4904 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:23:00.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.315367 4904 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:23:00.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.348604 4904 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:23:00.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.348920 4904 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:23:00.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.349408 4904 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:23:00.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.349747 4904 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:23:00.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.350042 4904 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:23:00.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.350347 4904 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:23:00.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.350572 4904 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40697/jwks.json
22:23:00.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:00.391144 4904 master_runner.cc:386] Master server non-default flags:
22:23:00.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:23:00.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.189.keytab
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.189
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.189:32841
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:23:00.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/info.pb
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.189
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37351
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/logs
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:23:00.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:23:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:00.393028 4904 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:23:00.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:00.394665 4904 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:23:00.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.415706 4911 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:23:01.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.416848 4910 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:23:01.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:00.416249 4913 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:23:01.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:01.577252 4912 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:23:01.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.577585 4904 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:23:01.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.582300 4904 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:23:01.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.585444 4904 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:23:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.586869 4904 hybrid_clock.cc:648] HybridClock initialized: now 1790461381586816 us; error 41 us; skew 500 ppm
22:23:01.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.587821 4904 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:23:01.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:23:01 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461381, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:23:01.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.599465 4904 init.cc:377] Logged in from keytab as kudu/127.4.111.189@KRBTEST.COM (short username kudu)
22:23:01.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.604894 4904 webserver.cc:460] Webserver started at http://127.4.111.189:37351/ using document root <none> and password file <none>
22:23:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.605888 4904 fs_manager.cc:359] Metadata directory not provided
22:23:01.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.606151 4904 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:23:01.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.615567 4904 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.003s sys 0.004s
22:23:01.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.620702 4923 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:23:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.622145 4904 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
22:23:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.622586 4904 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:23:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7d517c799d6b471abab944a9df73af84"
22:23:01.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:55 on dist-test-slave-tfj8"
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.625551 4904 fs_report.cc:383] FS layout report
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/data
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:23:01.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:23:01.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.681413 4904 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:23:01.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.688912 4904 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:23:01.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.689455 4904 kserver.cc:163] Server-wide thread pool size limit: 3276
22:23:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.770177 4904 rpc_server.cc:307] RPC server started. Bound to: 127.4.111.189:32841
22:23:01.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.770287 4974 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.111.189:32841 every 8 connection(s)
22:23:01.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.773660 4904 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/data/info.pb
22:23:01.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.781450 4542 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4904
22:23:01.783 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:23:01.783 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.4.111.188:44163
22:23:01.783 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
22:23:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.785300 4542 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:23:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu
22:23:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:23:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data
22:23:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
22:23:01.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/logs
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/info.pb
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.188:44163
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.188
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41285
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.188.keytab
22:23:01.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.188
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.785791 4975 sys_catalog.cc:263] Verifying existing consensus state
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 22:23:01.786727 4975 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
22:23:01.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.787196 4904 master.cc:537] Master@127.4.111.189:32841 shutting down...
22:23:01.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:01.878763 4904 master.cc:552] Master@127.4.111.189:32841 shutdown complete.
22:23:01.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
22:23:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.146555 4977 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
22:23:02.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.147387 4977 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
22:23:02.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.148175 4977 flags.cc:425] Enabled unsafe flag: --never_fsync=true
22:23:02.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.183817 4977 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
22:23:02.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.184082 4977 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
22:23:02.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.184427 4977 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
22:23:02.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.184640 4977 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
22:23:02.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.184849 4977 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
22:23:02.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.185040 4977 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
22:23:02.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.185184 4977 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40697/jwks.json
22:23:02.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:02.228365 4977 master_runner.cc:386] Master server non-default flags:
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.4.111.148:37529
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/krb5kdc/kudu_127.4.111.188.keytab
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.4.111.188
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.4.111.188:44163
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40697/jwks.json
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/info.pb
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.4.111.188
22:23:02.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41285
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/logs
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 21:14:29 UTC on ccycloud.quasar-htepcs.root.comops.site
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4675023
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
22:23:02.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:02.229915 4977 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
22:23:02.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:02.231031 4977 file_cache.cc:493] Constructed file cache file cache with capacity 419430
22:23:02.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.249749 4986 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:23:03.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.249902 4983 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:23:03.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:02.250030 4984 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
22:23:03.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:03.403591 4985 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
22:23:03.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.404278 4977 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
22:23:03.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.408644 4977 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
22:23:03.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.412520 4977 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
22:23:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.414165 4977 hybrid_clock.cc:648] HybridClock initialized: now 1790461383414079 us; error 63 us; skew 500 ppm
22:23:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.415206 4977 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
22:23:03.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:23:03 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461383, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:23:03.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.421592 4977 init.cc:377] Logged in from keytab as kudu/127.4.111.188@KRBTEST.COM (short username kudu)
22:23:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.428345 4977 webserver.cc:460] Webserver started at http://127.4.111.188:41285/ using document root <none> and password file <none>
22:23:03.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.429925 4977 fs_manager.cc:359] Metadata directory not provided
22:23:03.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.430244 4977 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
22:23:03.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.440954 4977 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
22:23:03.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.446344 4995 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
22:23:03.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.447763 4977 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.002s
22:23:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.448189 4977 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data,/tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:23:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dfbccaf74c56419c807a394e83895b60"
22:23:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 22:22:57 on dist-test-slave-tfj8"
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.450796 4977 fs_report.cc:383] FS layout report
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/wal
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/data
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
22:23:03.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
22:23:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
22:23:03.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
22:23:03.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.488431 4977 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
22:23:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.495213 4977 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
22:23:03.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.495734 4977 kserver.cc:163] Server-wide thread pool size limit: 3276
22:23:03.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.575884 4977 rpc_server.cc:307] RPC server started. Bound to: 127.4.111.188:44163
22:23:03.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.575943 5046 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.4.111.188:44163 every 8 connection(s)
22:23:03.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.579020 4977 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskmjfZXw/test-tmp/mini-kudu-cluster5964413549818914733/master-2/data/info.pb
22:23:03.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.588598 4542 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu as pid 4977
22:23:03.589 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
22:23:03.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.593226 5047 sys_catalog.cc:263] Verifying existing consensus state
22:23:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.602514 5047 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: Bootstrap starting.
22:23:03.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.652923 5047 log.cc:824] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: Log is configured to *not* fsync() on all Append() calls
22:23:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.659629 5047 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
22:23:03.660 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:03.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.660619 5047 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: Bootstrap complete.
22:23:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.699739 5047 raft_consensus.cc:348] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:23:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.703958 5047 raft_consensus.cc:374] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
22:23:03.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.704305 5047 raft_consensus.cc:729] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dfbccaf74c56419c807a394e83895b60, State: Initialized, Role: FOLLOWER
22:23:03.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.705775 5047 consensus_queue.cc:260] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:23:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.710516 5051 sys_catalog.cc:455] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } } }
22:23:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.711548 5051 sys_catalog.cc:458] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [sys.catalog]: This master's current role is: FOLLOWER
22:23:03.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.712836 5047 sys_catalog.cc:564] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [sys.catalog]: configured and running, proceeding with master startup.
22:23:03.744 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:03.747 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.4.111.188:44163(127.4.111.188:44163)] connecting to peer
22:23:03.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:03.751554 5001 tablet.cc:2369] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: Can't schedule compaction. Clean time has not been advanced past its initial value.
22:23:03.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:03.752996 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:03.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:03.753340 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:03.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.777796 5051 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:23:03.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.778298 5051 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:23:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.749719 4977 master_runner.cc:186] Error getting master registration for 127.4.111.190:39623: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111),
22:23:03.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.782512 4977 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:23:03.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:03.783668 4996 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111)
22:23:03.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.784608 5051 leader_election.cc:288] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623), 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841)
22:23:03.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:03.789462 4996 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623): Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111)
22:23:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:03.806860 4997 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841): Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:23:03.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.807343 4997 leader_election.cc:302] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbccaf74c56419c807a394e83895b60; no voters: 505d27ebcfa94756b3c137ba3013da8e, 7d517c799d6b471abab944a9df73af84
22:23:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:03.808218 5051 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:23:03.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:03.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:03.833 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:03.833 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:03.834 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:03.835 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:03.836 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.4.111.188:44163(127.4.111.188:44163)] Successfully connected to peer
22:23:03.927 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
22:23:03.938 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.4.111.188
22:23:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:23:04 dist-test-slave-tfj8 krb5kdc[4555](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461378, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.4.111.188@KRBTEST.COM
22:23:04.242 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x64145135, L:/127.0.0.1:40186 - R:/127.4.111.188:44163] using SASL/GSSAPI
22:23:04.265 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.277 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:04.280 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:04.283 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.283 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.284 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.286 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.287 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.287 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.293 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.316 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:04.318 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:04.319 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.319 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.321 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.324 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.324 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.329 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.357 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:04.358 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:04.360 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.361 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.362 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.363 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.364 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.364 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.366 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.376 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:04.377 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:04.380 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.380 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.383 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.385 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.386 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.386 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.387 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.416 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:04.417 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:04.419 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.419 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.420 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.426 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.427 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.429 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.496 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:04.497 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:04.500 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.501 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.503 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.505 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.507 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.556 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:04.559 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:04.560 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.560 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.561 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.564 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.564 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.565 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.567 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.616 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:04.618 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:04.619 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.619 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.622 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:04.624 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.624 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.625 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:04.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:04.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:04.756822 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:04.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:04.757125 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:05.016 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:05.018 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:05.020 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:05.020 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:05.021 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:05.021 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:05.022 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:05.022 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:05.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:05.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:05.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:05.539 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:05.539 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:05.539 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:05.540 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:05.540 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:05.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:05.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:05.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:05.760530 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:05.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:05.760829 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:05.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:05.894026 5070 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:23:05.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:05.894476 5070 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:23:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:05.896183 5070 leader_election.cc:288] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623), 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841)
22:23:05.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:05.902531 4996 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623): Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111)
22:23:05.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:05.902949 4997 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841): Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:23:05.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:05.903373 4997 leader_election.cc:302] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbccaf74c56419c807a394e83895b60; no voters: 505d27ebcfa94756b3c137ba3013da8e, 7d517c799d6b471abab944a9df73af84
22:23:05.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:05.904182 5070 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:23:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:06.764195 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:06.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:06.764485 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:07.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:07.017518 4048 debug-util.cc:398] Leaking SignalData structure 0x7b080009cee0 after lost signal to thread 3980
22:23:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:07.018244 4048 debug-util.cc:398] Leaking SignalData structure 0x7b08000afe40 after lost signal to thread 4051
22:23:07.156 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:07.159 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:07.159 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:07.159 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:07.160 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:07.165 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:07.165 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:07.165 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:07.167 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:07.276 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:07.277 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:07.278 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:07.278 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:07.279 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:07.281 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:07.281 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:07.281 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:07.284 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:07.767616 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:07.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:07.767872 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:08.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:08.612689 5078 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:23:08.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:08.613204 5078 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:23:08.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:08.614919 5078 leader_election.cc:288] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623), 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841)
22:23:08.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:08.620210 4996 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623): Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111)
22:23:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:08.620716 4997 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841): Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:23:08.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:08.621122 4997 leader_election.cc:302] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbccaf74c56419c807a394e83895b60; no voters: 505d27ebcfa94756b3c137ba3013da8e, 7d517c799d6b471abab944a9df73af84
22:23:08.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:08.621804 5078 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:23:08.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:08.770840 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:08.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:08.771096 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:08.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:08.784987 4977 master_runner.cc:186] Error getting master registration for 127.4.111.190:39623: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111),
22:23:08.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:08.785411 4977 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:23:09.556 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:09.556 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:09.558 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:09.558 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:09.558 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:09.560 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:09.560 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:09.560 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:09.563 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:09.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:09.774272 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:09.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:09.774559 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:09.936 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:09.937 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:09.939 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:09.939 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:09.940 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:09.940 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:09.940 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:09.941 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:09.943 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:10.777972 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:10.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:10.778292 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:11.575829 5043 debug-util.cc:398] Leaking SignalData structure 0x7b08000b09a0 after lost signal to thread 4978
22:23:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:11.576561 5043 debug-util.cc:398] Leaking SignalData structure 0x7b080009c860 after lost signal to thread 5046
22:23:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:11.782747 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:11.783142 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:11.867733 5085 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:23:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:11.868178 5085 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:23:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:11.869771 5085 leader_election.cc:288] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623), 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841)
22:23:11.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:11.871840 4996 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111) [suppressed 5 similar messages]
22:23:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:11.874771 4996 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623): Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111)
22:23:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:11.875593 4997 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841): Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:23:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:11.875880 4997 leader_election.cc:302] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbccaf74c56419c807a394e83895b60; no voters: 505d27ebcfa94756b3c137ba3013da8e, 7d517c799d6b471abab944a9df73af84
22:23:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:11.876511 5085 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:23:12.116 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:12.117 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:12.119 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:12.119 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:12.119 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:12.120 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:12.120 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:12.120 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:12.139 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:12.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:12.787248 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:12.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:12.787705 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:23:13 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461393, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:23:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:13.426339 4991 init.cc:295] Successfully reacquired a new kerberos TGT
22:23:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:13.790529 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:13.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:13.790791 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:13.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:13.826149 4977 master_runner.cc:186] Error getting master registration for 127.4.111.190:39623: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111),
22:23:13.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:13.826711 4977 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:23:14.176 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:14.178 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:14.179 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:14.180 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:14.180 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:14.182 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:14.182 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:14.182 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:14.184 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:14.793958 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:14.794216 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:15.797901 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:15.798198 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:16.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:16.801575 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:16.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:16.801928 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:17.805305 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:17.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:17.805637 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:17.976 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:17.977 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:17.979 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:17.979 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:17.979 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:17.980 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:17.980 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:17.980 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:17.984 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:18.810082 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:18.810462 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:18.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:18.829869 4977 master_runner.cc:186] Error getting master registration for 127.4.111.190:39623: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111),
22:23:18.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:18.830387 4977 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:23:19.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:19.171026 4246 debug-util.cc:398] Leaking SignalData structure 0x7b08000afb60 after lost signal to thread 4119
22:23:19.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:19.172372 4246 debug-util.cc:398] Leaking SignalData structure 0x7b08000bd080 after lost signal to thread 4249
22:23:19.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:19.363945 5098 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:23:19.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:19.364497 5098 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:23:19.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:19.366132 5098 leader_election.cc:288] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623), 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841)
22:23:19.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:19.369048 4996 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111) [suppressed 1 similar messages]
22:23:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:19.372565 4997 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841): Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:23:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:19.373114 4996 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623): Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111)
22:23:19.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:19.373497 4996 leader_election.cc:302] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbccaf74c56419c807a394e83895b60; no voters: 505d27ebcfa94756b3c137ba3013da8e, 7d517c799d6b471abab944a9df73af84
22:23:19.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:19.374243 5098 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:23:19.496 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:19.497 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:19.499 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:19.500 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:19.500 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:19.502 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:19.502 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:19.503 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:19.503 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:19.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:19.814833 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:19.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:19.815204 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:20.818140 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:20.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:20.818401 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:21.796 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:21.797 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:21.799 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:21.799 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:21.799 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:21.800 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:21.800 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:21.800 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:21.804 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:21.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:21.818763 5104 raft_consensus.cc:482] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
22:23:21.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:21.819275 5104 raft_consensus.cc:504] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "505d27ebcfa94756b3c137ba3013da8e" member_type: VOTER last_known_addr { host: "127.4.111.190" port: 39623 } } peers { permanent_uuid: "7d517c799d6b471abab944a9df73af84" member_type: VOTER last_known_addr { host: "127.4.111.189" port: 32841 } } peers { permanent_uuid: "dfbccaf74c56419c807a394e83895b60" member_type: VOTER last_known_addr { host: "127.4.111.188" port: 44163 } }
22:23:21.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:21.821697 5104 leader_election.cc:288] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623), 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841)
22:23:21.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:21.829423 4996 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 505d27ebcfa94756b3c137ba3013da8e (127.4.111.190:39623): Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111)
22:23:21.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:21.829458 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:21.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:21.830351 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:21.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:21.830354 4997 leader_election.cc:334] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 7d517c799d6b471abab944a9df73af84 (127.4.111.189:32841): Network error: Client connection negotiation failed: client connection to 127.4.111.189:32841: connect: Connection refused (error 111)
22:23:21.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:21.830825 4997 leader_election.cc:302] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dfbccaf74c56419c807a394e83895b60; no voters: 505d27ebcfa94756b3c137ba3013da8e, 7d517c799d6b471abab944a9df73af84
22:23:21.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:21.831743 5104 raft_consensus.cc:2726] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
22:23:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:22.834587 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:22.834997 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:23.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:23.219439 3969 debug-util.cc:398] Leaking SignalData structure 0x7b08000ab000 after lost signal to thread 3904
22:23:23.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:23.220247 3969 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae720 after lost signal to thread 3972
22:23:23.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 22:23:23 dist-test-slave-tfj8 krb5kdc[4555](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790461403, etypes {rep=17 tkt=17 ses=17}, kudu/127.4.111.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
22:23:23.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:23.429375 4991 init.cc:295] Successfully reacquired a new kerberos TGT
22:23:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:23.839407 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:23.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:23.840276 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:23.841918 4977 master_runner.cc:186] Error getting master registration for 127.4.111.190:39623: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111),
22:23:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:23.842494 4977 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:23:23.977 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:23.977 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:23.978 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:23.978 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:23.978 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:23.979 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:23.979 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:23.980 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:23.985 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:24.843989 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:24.844319 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:25.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:25.847579 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:25.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:25.847877 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:26.850695 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:26.850951 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:27.576 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:27.577 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:27.577 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:27.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:27.578 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:27.580 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:27.580 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:27.580 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:27.582 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:27.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:27.854074 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:27.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:27.854365 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:28.845714 4977 master_runner.cc:186] Error getting master registration for 127.4.111.190:39623: Network error: Client connection negotiation failed: client connection to 127.4.111.190:39623: connect: Connection refused (error 111),
22:23:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:28.846145 4977 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
22:23:28.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:28.857601 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:28.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:28.857906 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:29.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:29.860945 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:29.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:29.861198 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:30.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:30.864564 5062 catalog_manager.cc:1558] T 00000000000000000000000000000000 P dfbccaf74c56419c807a394e83895b60: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
22:23:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 22:23:30.864810 5062 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
22:23:31.476 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.190:39623(127.4.111.190:39623)] connecting to peer
22:23:31.477 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.4.111.189:32841(127.4.111.189:32841)] connecting to peer
22:23:31.478 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:31.478 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.4.111.190:39623(127.4.111.190:39623)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:31.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.190:39623: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623
22:23:31.478 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:31.478 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.189:32841(127.4.111.189:32841)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:31.479 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.4.111.189:32841: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
22:23:31.481 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163; will retry
22:23:31.486 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27887), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [230ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [235ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [659ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [687ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [692ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [694ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [720ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [725ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [728ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [761ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [764ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [766ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [780ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [785ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [786ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [787ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [820ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [826ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [828ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [895ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [900ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [903ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [906ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [909ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [960ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [965ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [966ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1016ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1026ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1027ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1415ms] refreshing cache from master, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1429ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1431ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1935ms] refreshing cache from master, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1939ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1940ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1943ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1943ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3555ms] refreshing cache from master, [3555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [3558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [3559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [3560ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [3565ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3566ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [3567ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3675ms] refreshing cache from master, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [3677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [3678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [3679ms] too many traces: truncated at 100 traces, deferred=Deferred@555170259(state=PENDING, result=null, callback=(continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918) -> (continuation of Deferred@1230777150 after retry RPC after error@451906515), errback=(continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918) -> (continuation of Deferred@1230777150 after retry RPC after error@451906515)))
22:23:31.492 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.111.190:39623(127.4.111.190:39623)] disconnecting while in state TERMINATED
22:23:31.494 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.111.189:32841(127.4.111.189:32841)] disconnecting while in state TERMINATED
22:23:31.494 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.4.111.188:44163(127.4.111.188:44163)] disconnecting while in state READY
22:23:31.501 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:23:31.503 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.4.111.188:44163(127.4.111.188:44163)] handling channelInactive
22:23:31.503 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.4.111.188:44163(127.4.111.188:44163)] cleaning up while in state READY due to: connection closed
22:23:31.505 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:23:31.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:31.509573 4542 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 4977
22:23:32.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T22:23:32Z chronyd exiting
22:23:33.639 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
22:23:33.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:33.641707 3818 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 4098
22:23:33.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:33.668147 3818 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 4252
22:23:33.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:33.692440 3818 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 4399
22:23:33.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:33.720424 3818 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 3830
22:23:33.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:33.748903 3818 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 3900
22:23:33.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 22:23:33.784394 3818 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskmjfZXw/build/tsan/bin/kudu with pid 3975
22:23:34.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T22:23:34Z chronyd exiting
22:23:35.896 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27887), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [230ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [235ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [659ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [687ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [692ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [694ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [720ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [725ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [728ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [761ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [764ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [766ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [780ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [785ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [786ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [787ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [820ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [826ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [828ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [895ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [900ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [903ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [906ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [909ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [960ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [965ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [966ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1016ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1026ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1027ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1415ms] refreshing cache from master, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1429ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1431ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1935ms] refreshing cache from master, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1939ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1940ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1943ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1943ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3555ms] refreshing cache from master, [3555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [3558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [3559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [3560ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [3565ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3566ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [3567ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3675ms] refreshing cache from master, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [3677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [3678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [3679ms] too many traces: truncated at 100 traces, deferred=Deferred@555170259(state=PENDING, result=null, callback=(continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918) -> (continuation of Deferred@1230777150 after retry RPC after error@451906515), errback=(continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918) -> (continuation of Deferred@1230777150 after retry RPC after error@451906515)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.111.189:32841
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27887), Traces: [0ms] refreshing cache from master, [48ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [146ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [230ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [235ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [659ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [666ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [683ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [683ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [687ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [692ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [694ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [716ms] refreshing cache from master, [716ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [719ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [720ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [725ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [728ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [729ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [756ms] refreshing cache from master, [756ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [758ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [760ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [761ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [764ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [766ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [767ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [775ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [780ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [785ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [786ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [787ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [815ms] refreshing cache from master, [816ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [820ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [826ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [828ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [830ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [895ms] refreshing cache from master, [896ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [899ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [900ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [903ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [906ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [909ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [955ms] refreshing cache from master, [956ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [960ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [960ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [965ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [966ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [968ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1016ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1018ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1019ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1024ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1026ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1027ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1415ms] refreshing cache from master, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1420ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1429ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1431ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1935ms] refreshing cache from master, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [1936ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [1938ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [1939ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [1940ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [1943ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [1943ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3555ms] refreshing cache from master, [3555ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [3558ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [3559ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [3560ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.190:39623: Network error: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623, [3565ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.189:32841: Network error: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3566ms] Sub RPC ConnectToMaster: received response from server master-127.4.111.188:44163: OK, [3567ms] delaying RPC due to: Service unavailable: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841, [3675ms] refreshing cache from master, [3676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.190:39623, [3677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.189:32841, [3678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.4.111.188:44163, [3679ms] too many traces: truncated at 100 traces, deferred=Deferred@555170259(state=PENDING, result=null, callback=(continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918) -> (continuation of Deferred@1230777150 after retry RPC after error@451906515), errback=(continuation of Deferred@843418037 after retry RPC after error@52428245) -> (continuation of Deferred@895112287 after retry RPC after error@756755078) -> (continuation of Deferred@856495793 after retry RPC after error@1325442128) -> (continuation of Deferred@673184067 after retry RPC after error@1545910748) -> (continuation of Deferred@122127479 after retry RPC after error@1294959339) -> (continuation of Deferred@447594476 after retry RPC after error@1673400124) -> (continuation of Deferred@1491159808 after retry RPC after error@1776857645) -> (continuation of Deferred@516760099 after retry RPC after error@219748649) -> (continuation of Deferred@1601370817 after retry RPC after error@1940345560) -> (continuation of Deferred@76327605 after retry RPC after error@1092168540) -> (continuation of Deferred@158692684 after retry RPC after error@1083033384) -> (continuation of Deferred@557126798 after retry RPC after error@704550975) -> (continuation of Deferred@184764249 after retry RPC after error@630810006) -> (continuation of Deferred@746685647 after retry RPC after error@1910758912) -> (continuation of Deferred@1615554282 after retry RPC after error@244325024) -> (continuation of Deferred@773279809 after retry RPC after error@1097566301) -> (continuation of Deferred@2058148524 after retry RPC after error@1961331267) -> (continuation of Deferred@1617316294 after retry RPC after error@2110917879) -> (continuation of Deferred@1639436379 after retry RPC after error@692775868) -> (continuation of Deferred@2066788138 after retry RPC after error@384348431) -> (continuation of Deferred@1233297025 after retry RPC after error@929798918) -> (continuation of Deferred@1230777150 after retry RPC after error@451906515)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.4.111.190:39623,127.4.111.189:32841,127.4.111.188:44163) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.190:39623(127.4.111.190:39623): Connection refused: /127.4.111.190:39623,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.4.111.189:32841(127.4.111.189:32841): Connection refused: /127.4.111.189:32841
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.4.111.189:32841
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)