20:25:50.573 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:25:50.585 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskIlcT6h/test-tmp
20:25:50.588 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:25:50.591 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskIlcT6h/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:25:50.866 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:25:51.039 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:25:51.040 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:25:51.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.044274 28002 mini_oidc.cc:152] Starting JWKS server
20:25:51.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.048137 28002 webserver.cc:460] Webserver started at http://127.0.0.1:42573/ using document root <none> and password file <none>
20:25:51.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.048741 28002 mini_oidc.cc:189] Starting OIDC Discovery server
20:25:51.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.050868 28002 webserver.cc:460] Webserver started at http://127.0.0.1:40879/ using document root <none> and password file <none>
20:25:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:25:51Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:25:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:25:51Z Disabled control of system clock
20:25:51.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.108270 28002 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:25:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:25:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/wal
20:25:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/data
20:25:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:25:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:25:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:25:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:25:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:25:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:25:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/logs
20:25:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/data/info.pb
20:25:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:25:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.190:38777
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.190
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:25:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:25:51.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:25:51.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42573/jwks.json with env {}
20:25:51.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.416239 28020 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:25:51.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.416955 28020 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:25:51.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.417479 28020 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:25:51.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.448659 28020 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:25:51.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.448946 28020 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:25:51.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.449138 28020 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:25:51.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.449354 28020 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:25:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.449566 28020 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:25:51.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.449736 28020 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42573/jwks.json
20:25:51.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.493106 28020 master_runner.cc:386] Master server non-default flags:
20:25:51.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:25:51.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:25:51.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:25:51.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:25:51.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/data
20:25:51.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/wal
20:25:51.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:25:51.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.190:38777
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42573/jwks.json
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/data/info.pb
20:25:51.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.190
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/logs
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:25:51.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:25:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:25:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:25:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:25:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:25:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:25:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.494585 28020 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:25:51.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.496752 28020 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:25:51.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.516670 28026 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:52.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.516726 28030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:52.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:51.518201 28027 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:52.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:51.518338 28020 server_base.cc:1034] running on GCE node
20:25:52.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.623461 28020 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:25:52.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.626124 28020 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:25:52.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.627606 28020 hybrid_clock.cc:648] HybridClock initialized: now 1786047952627602 us; error 148 us; skew 500 ppm
20:25:52.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.628463 28020 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:25:52.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.634842 28020 webserver.cc:460] Webserver started at http://127.27.88.190:32883/ using document root <none> and password file <none>
20:25:52.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.635807 28020 fs_manager.cc:359] Metadata directory not provided
20:25:52.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.636000 28020 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:25:52.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.636394 28020 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:25:52.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.641481 28020 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/data/instance:
20:25:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2b7323b84d84dbb86a46e1aca201581"
20:25:52.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:52 on dist-test-slave-jpj1"
20:25:52.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.642731 28020 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/wal/instance:
20:25:52.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2b7323b84d84dbb86a46e1aca201581"
20:25:52.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:52 on dist-test-slave-jpj1"
20:25:52.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.650466 28020 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
20:25:52.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.655993 28040 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:25:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.657054 28020 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:25:52.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.657413 28020 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/wal
20:25:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d2b7323b84d84dbb86a46e1aca201581"
20:25:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:52 on dist-test-slave-jpj1"
20:25:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.657796 28020 fs_report.cc:383] FS layout report
20:25:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:25:52.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/wal
20:25:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/wal
20:25:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/data/data
20:25:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:25:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:25:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:25:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:25:52.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:25:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:25:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:25:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:25:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:25:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:25:52.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:25:52.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.690745 28020 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:25:52.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.692405 28020 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:25:52.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.692902 28020 kserver.cc:163] Server-wide thread pool size limit: 3276
20:25:52.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.760473 28020 rpc_server.cc:307] RPC server started. Bound to: 127.27.88.190:38777
20:25:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.760563 28109 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.88.190:38777 every 8 connection(s)
20:25:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.763496 28020 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/data/info.pb
20:25:52.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.768752 28110 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:25:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.774189 28002 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 28020
20:25:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.774653 28002 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-0/wal/instance
20:25:52.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.778481 28002 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/wal
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/data
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/logs
20:25:52.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/data/info.pb
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:25:52.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.189:37063
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.189
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:25:52.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:25:52.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:25:52.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42573/jwks.json with env {}
20:25:52.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.780750 28110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } has no permanent_uuid. Determining permanent_uuid...
20:25:52.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.808508 28110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } has no permanent_uuid. Determining permanent_uuid...
20:25:52.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:52.810643 28041 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.88.189:37063: connect: Connection refused (error 111)
20:25:52.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:52.814558 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.189:37063: Network error: Client connection negotiation failed: client connection to 127.27.88.189:37063: connect: Connection refused (error 111)
20:25:52.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.864117 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } attempt: 1
20:25:52.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:52.867960 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.189:37063: Network error: Client connection negotiation failed: client connection to 127.27.88.189:37063: connect: Connection refused (error 111)
20:25:52.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:52.936404 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } attempt: 2
20:25:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:52.942018 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.189:37063: Network error: Client connection negotiation failed: client connection to 127.27.88.189:37063: connect: Connection refused (error 111)
20:25:53.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:53.033475 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } attempt: 3
20:25:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.038022 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.189:37063: Network error: Client connection negotiation failed: client connection to 127.27.88.189:37063: connect: Connection refused (error 111)
20:25:53.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.093286 28115 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:25:53.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.093863 28115 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:25:53.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.094383 28115 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:25:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.124845 28115 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:25:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.125128 28115 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:25:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.125313 28115 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:25:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.125490 28115 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:25:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.125672 28115 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:25:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.125857 28115 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42573/jwks.json
20:25:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:53.160990 28115 master_runner.cc:386] Master server non-default flags:
20:25:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:25:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:25:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:25:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:25:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/data
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/wal
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.189:37063
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42573/jwks.json
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:25:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/data/info.pb
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.189
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/logs
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:25:53.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:25:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:25:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:25:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:25:53.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:25:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:25:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:25:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:53.162230 28115 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:25:53.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:53.164458 28115 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:25:53.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:53.181393 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } attempt: 4
20:25:53.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.185423 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.189:37063: Network error: Client connection negotiation failed: client connection to 127.27.88.189:37063: connect: Connection refused (error 111)
20:25:53.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.188232 28129 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:53.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:53.484928 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } attempt: 5
20:25:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.490099 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.189:37063: Network error: Client connection negotiation failed: client connection to 127.27.88.189:37063: connect: Connection refused (error 111)
20:25:54.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.037657 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } attempt: 6
20:25:54.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.047767 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.189:37063: Network error: Client connection negotiation failed: client connection to 127.27.88.189:37063: connect: Connection refused (error 111)
20:25:54.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.188267 28128 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:54.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:53.188243 28131 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.348770 28130 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1158 milliseconds
20:25:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.348891 28115 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:25:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.350543 28115 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:25:54.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.353601 28115 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:25:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.355021 28115 hybrid_clock.cc:648] HybridClock initialized: now 1786047954354979 us; error 61 us; skew 500 ppm
20:25:54.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.355872 28115 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:25:54.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.362970 28115 webserver.cc:460] Webserver started at http://127.27.88.189:36317/ using document root <none> and password file <none>
20:25:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.363994 28115 fs_manager.cc:359] Metadata directory not provided
20:25:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.364178 28115 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:25:54.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.364609 28115 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:25:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.370009 28115 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/data/instance:
20:25:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "03012590d64e4889a32c303cf20e8db4"
20:25:54.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:54 on dist-test-slave-jpj1"
20:25:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.371178 28115 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/wal/instance:
20:25:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "03012590d64e4889a32c303cf20e8db4"
20:25:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:54 on dist-test-slave-jpj1"
20:25:54.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.380152 28115 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
20:25:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.385933 28143 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:25:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.386999 28115 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
20:25:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.387367 28115 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/wal
20:25:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "03012590d64e4889a32c303cf20e8db4"
20:25:54.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:54 on dist-test-slave-jpj1"
20:25:54.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.387763 28115 fs_report.cc:383] FS layout report
20:25:54.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:25:54.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/wal
20:25:54.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/wal
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/data/data
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:25:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:25:54.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.433485 28115 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:25:54.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.435182 28115 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:25:54.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.435804 28115 kserver.cc:163] Server-wide thread pool size limit: 3276
20:25:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.517310 28115 rpc_server.cc:307] RPC server started. Bound to: 127.27.88.189:37063
20:25:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.517386 28215 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.88.189:37063 every 8 connection(s)
20:25:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.520102 28115 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/data/info.pb
20:25:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.522730 28002 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 28115
20:25:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.523200 28002 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-1/wal/instance
20:25:54.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.526435 28216 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.526640 28002 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/wal
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/data
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/logs
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/data/info.pb
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:25:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.188:46035
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.188
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:25:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42573/jwks.json with env {}
20:25:54.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.541018 28216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } has no permanent_uuid. Determining permanent_uuid...
20:25:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.565157 28216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } has no permanent_uuid. Determining permanent_uuid...
20:25:54.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.581017 28216 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } has no permanent_uuid. Determining permanent_uuid...
20:25:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.584100 28150 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:54.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.588433 28216 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:54.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.637948 28216 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 1
20:25:54.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.642021 28216 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:54.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.710479 28216 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 2
20:25:54.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.714463 28216 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:54.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.805874 28216 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 3
20:25:54.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.809762 28216 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:54.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.859171 28218 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:25:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.859910 28218 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:25:54.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.860566 28218 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:25:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.891065 28218 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:25:54.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.891414 28218 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:25:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.891784 28218 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:25:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.892092 28218 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:25:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.892424 28218 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:25:54.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.892642 28218 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42573/jwks.json
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.927517 28218 master_runner.cc:386] Master server non-default flags:
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/data
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/wal
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:25:54.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.188:46035
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42573/jwks.json
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/data/info.pb
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.188
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/logs
20:25:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:25:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:25:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:25:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:25:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:25:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:25:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:25:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:25:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:25:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:25:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.929129 28218 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:25:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.931411 28218 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:25:54.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.950225 28235 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:54.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:54.953116 28216 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 4
20:25:54.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.957062 28216 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.108291 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } attempt: 7
20:25:55.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.121420 28110 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } has no permanent_uuid. Determining permanent_uuid...
20:25:55.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:55.127197 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:55.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.185573 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 1
20:25:55.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:55.189795 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.256489 28216 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 5
20:25:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:55.260190 28216 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:55.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.271162 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 2
20:25:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:55.275430 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.360875 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 3
20:25:55.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:55.365340 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:55.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.505734 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 4
20:25:55.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:55.509816 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:55.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.793357 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 5
20:25:55.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:55.797199 28110 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:55.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:55.807725 28216 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 6
20:25:55.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:55.814435 28216 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.27.88.188:46035: Network error: Client connection negotiation failed: client connection to 127.27.88.188:46035: connect: Connection refused (error 111)
20:25:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.950453 28231 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:54.950434 28233 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:56.111583 28234 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:25:56.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.111722 28218 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:25:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.115746 28218 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:25:56.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.120675 28218 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:25:56.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.122148 28218 hybrid_clock.cc:648] HybridClock initialized: now 1786047956122107 us; error 60 us; skew 500 ppm
20:25:56.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.123049 28218 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:25:56.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.136588 28218 webserver.cc:460] Webserver started at http://127.27.88.188:38439/ using document root <none> and password file <none>
20:25:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.137916 28218 fs_manager.cc:359] Metadata directory not provided
20:25:56.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.138201 28218 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:25:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.138787 28218 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:25:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.144770 28218 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/data/instance:
20:25:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85064d43662b4271b17b747916fe3ba7"
20:25:56.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:56 on dist-test-slave-jpj1"
20:25:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.146234 28218 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/wal/instance:
20:25:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85064d43662b4271b17b747916fe3ba7"
20:25:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:56 on dist-test-slave-jpj1"
20:25:56.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.154068 28218 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
20:25:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.160778 28250 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:25:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.161832 28218 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
20:25:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.162185 28218 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/wal
20:25:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85064d43662b4271b17b747916fe3ba7"
20:25:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:56 on dist-test-slave-jpj1"
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.162531 28218 fs_report.cc:383] FS layout report
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/wal
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/wal
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/data/data
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:25:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:25:56.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.200017 28218 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:25:56.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.201761 28218 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:25:56.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.202286 28218 kserver.cc:163] Server-wide thread pool size limit: 3276
20:25:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.281030 28218 rpc_server.cc:307] RPC server started. Bound to: 127.27.88.188:46035
20:25:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.281132 28327 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.88.188:46035 every 8 connection(s)
20:25:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.283975 28218 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/data/info.pb
20:25:56.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.284565 28002 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 28218
20:25:56.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.285023 28002 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/master-2/wal/instance
20:25:56.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.295269 28328 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:25:56.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.314694 28328 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } has no permanent_uuid. Determining permanent_uuid...
20:25:56.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.342588 28328 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } has no permanent_uuid. Determining permanent_uuid...
20:25:56.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.349671 28110 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 6
20:25:56.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.356499 28328 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } has no permanent_uuid. Determining permanent_uuid...
20:25:56.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.379077 28110 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: Bootstrap starting.
20:25:56.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.383590 28328 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7: Bootstrap starting.
20:25:56.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.386341 28110 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: Neither blocks nor log segments found. Creating new log.
20:25:56.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.388836 28110 log.cc:824] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: Log is configured to *not* fsync() on all Append() calls
20:25:56.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.391353 28328 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7: Neither blocks nor log segments found. Creating new log.
20:25:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.393625 28328 log.cc:824] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7: Log is configured to *not* fsync() on all Append() calls
20:25:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.394615 28110 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: No bootstrap required, opened a new log
20:25:56.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.398528 28328 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7: No bootstrap required, opened a new log
20:25:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.418684 28110 raft_consensus.cc:348] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.419317 28110 raft_consensus.cc:374] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:25:56.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.419504 28110 raft_consensus.cc:729] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d2b7323b84d84dbb86a46e1aca201581, State: Initialized, Role: FOLLOWER
20:25:56.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.420523 28110 consensus_queue.cc:260] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.421952 28328 raft_consensus.cc:348] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:56.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.423029 28328 raft_consensus.cc:374] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:25:56.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.423511 28328 raft_consensus.cc:729] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 85064d43662b4271b17b747916fe3ba7, State: Initialized, Role: FOLLOWER
20:25:56.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.423988 28344 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.425107 28344 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [sys.catalog]: This master's current role is: FOLLOWER
20:25:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.424824 28328 consensus_queue.cc:260] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:56.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.425761 28110 sys_catalog.cc:564] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [sys.catalog]: configured and running, proceeding with master startup.
20:25:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.429520 28347 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:56.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.431490 28347 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: This master's current role is: FOLLOWER
20:25:56.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.431990 28328 sys_catalog.cc:564] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: configured and running, proceeding with master startup.
20:25:56.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:56.452637 28047 tablet.cc:2369] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:25:56.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:56.457760 28258 tablet.cc:2369] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:25:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:56.472904 28377 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:25:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:56.473604 28377 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:25:56.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.462031 28218 master_runner.cc:186] Error getting master registration for 127.27.88.190:38777: OK, instance_id { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" instance_seqno: 1786047952742074 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:25:56.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.494987 28218 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:25:56.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:56.495132 28379 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:25:56.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:56.495450 28379 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:25:56.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.471678 28020 master_runner.cc:186] Error getting master registration for 127.27.88.189:37063: OK, instance_id { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" instance_seqno: 1786047954496290 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:25:56.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.507529 28020 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:25:56.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.603268 28347 raft_consensus.cc:482] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:25:56.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.603911 28347 raft_consensus.cc:504] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:56.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.610144 28076 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "85064d43662b4271b17b747916fe3ba7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d2b7323b84d84dbb86a46e1aca201581" is_pre_election: true
20:25:56.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.611182 28347 leader_election.cc:288] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d2b7323b84d84dbb86a46e1aca201581 (127.27.88.190:38777), 03012590d64e4889a32c303cf20e8db4 (127.27.88.189:37063)
20:25:56.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.611449 28076 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 85064d43662b4271b17b747916fe3ba7 in term 0.
20:25:56.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.612910 28256 leader_election.cc:302] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 85064d43662b4271b17b747916fe3ba7, d2b7323b84d84dbb86a46e1aca201581; no voters:
20:25:56.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.614115 28347 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 0 FOLLOWER]: Leader pre-election won for term 1
20:25:56.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.614665 28347 raft_consensus.cc:482] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:25:56.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.615181 28347 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 0 FOLLOWER]: Advancing to term 1
20:25:56.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.622097 28347 raft_consensus.cc:504] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.624974 28076 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "85064d43662b4271b17b747916fe3ba7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d2b7323b84d84dbb86a46e1aca201581"
20:25:56.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.625553 28076 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [term 0 FOLLOWER]: Advancing to term 1
20:25:56.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.624519 28347 leader_election.cc:288] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [CANDIDATE]: Term 1 election: Requested vote from peers d2b7323b84d84dbb86a46e1aca201581 (127.27.88.190:38777), 03012590d64e4889a32c303cf20e8db4 (127.27.88.189:37063)
20:25:56.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.641568 28076 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 85064d43662b4271b17b747916fe3ba7 in term 1.
20:25:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.651048 28256 leader_election.cc:302] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 85064d43662b4271b17b747916fe3ba7, d2b7323b84d84dbb86a46e1aca201581; no voters:
20:25:56.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.651983 28347 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 1 FOLLOWER]: Leader election won for term 1
20:25:56.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.654883 28347 raft_consensus.cc:686] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [term 1 LEADER]: Becoming Leader. State: Replica: 85064d43662b4271b17b747916fe3ba7, State: Running, Role: LEADER
20:25:56.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.656148 28347 consensus_queue.cc:237] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:56.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.677733 28382 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 85064d43662b4271b17b747916fe3ba7. Latest consensus state: current_term: 1 leader_uuid: "85064d43662b4271b17b747916fe3ba7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:56.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.679004 28382 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: This master's current role is: LEADER
20:25:56.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.687728 28384 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
20:25:56.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.697433 28384 catalog_manager.cc:1484] Initializing Kudu cluster ID...
20:25:56.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.725589 28076 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [term 1 FOLLOWER]: Refusing update from remote peer 85064d43662b4271b17b747916fe3ba7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:25:56.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.728515 28382 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:25:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.757220 28344 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 85064d43662b4271b17b747916fe3ba7. Latest consensus state: current_term: 1 leader_uuid: "85064d43662b4271b17b747916fe3ba7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:56.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.757934 28344 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [sys.catalog]: This master's current role is: FOLLOWER
20:25:56.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.769753 28385 mvcc.cc:205] Tried to move back new op lower bound from 7315652430722572288 to 7315652430474547200. Current Snapshot: MvccSnapshot[applied={T|T < 7315652430722572288}]
20:25:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.774863 28347 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "85064d43662b4271b17b747916fe3ba7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.775637 28347 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: This master's current role is: LEADER
20:25:56.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.788219 28384 catalog_manager.cc:1347] Generated new cluster ID: c7e35025abdc48d3be8d21fa12b960c1
20:25:56.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.788523 28384 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
20:25:56.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.790704 28344 sys_catalog.cc:455] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "85064d43662b4271b17b747916fe3ba7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:56.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.792505 28344 sys_catalog.cc:458] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581 [sys.catalog]: This master's current role is: FOLLOWER
20:25:56.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.832844 28384 catalog_manager.cc:1370] Generated new certificate authority record
20:25:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.835485 28384 catalog_manager.cc:1504] Loading token signing keys...
20:25:56.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.874989 28216 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } attempt: 7
20:25:56.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.892462 28384 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7: Generated new TSK 0
20:25:56.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.898173 28384 catalog_manager.cc:1514] Initializing in-progress tserver states...
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.927700 28002 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/wal
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/data
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/logs
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/data/info.pb
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:25:56.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.129:0
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.88.129
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.129
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:25:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:25:56.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.951357 28216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4: Bootstrap starting.
20:25:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.969700 28216 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4: Neither blocks nor log segments found. Creating new log.
20:25:56.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.975749 28216 log.cc:824] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4: Log is configured to *not* fsync() on all Append() calls
20:25:57.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:56.999195 28216 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4: No bootstrap required, opened a new log
20:25:57.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.084525 28216 raft_consensus.cc:348] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.086735 28216 raft_consensus.cc:374] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:25:57.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.087496 28216 raft_consensus.cc:729] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 03012590d64e4889a32c303cf20e8db4, State: Initialized, Role: FOLLOWER
20:25:57.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.089803 28216 consensus_queue.cc:260] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } }
20:25:57.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.104640 28397 sys_catalog.cc:455] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.105528 28397 sys_catalog.cc:458] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [sys.catalog]: This master's current role is: FOLLOWER
20:25:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.111655 28181 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [term 0 FOLLOWER]: Advancing to term 1
20:25:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.115845 28184 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "85064d43662b4271b17b747916fe3ba7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "03012590d64e4889a32c303cf20e8db4"
20:25:57.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.118283 28185 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "85064d43662b4271b17b747916fe3ba7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "03012590d64e4889a32c303cf20e8db4" is_pre_election: true
20:25:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.124969 28181 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [term 1 FOLLOWER]: Refusing update from remote peer 85064d43662b4271b17b747916fe3ba7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:25:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.126384 28216 sys_catalog.cc:564] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [sys.catalog]: configured and running, proceeding with master startup.
20:25:57.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.126734 28347 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:25:57.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.204010 28397 sys_catalog.cc:455] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 85064d43662b4271b17b747916fe3ba7. Latest consensus state: current_term: 1 leader_uuid: "85064d43662b4271b17b747916fe3ba7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:57.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.204331 28404 sys_catalog.cc:455] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "85064d43662b4271b17b747916fe3ba7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:57.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.204931 28397 sys_catalog.cc:458] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [sys.catalog]: This master's current role is: FOLLOWER
20:25:57.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.206489 28404 sys_catalog.cc:458] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4 [sys.catalog]: This master's current role is: FOLLOWER
20:25:57.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.243108 28422 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:25:57.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:57.486907 28395 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:25:57.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:57.487769 28395 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:25:57.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:57.488567 28395 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:25:57.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.502575 28379 catalog_manager.cc:1259] Loaded cluster ID: c7e35025abdc48d3be8d21fa12b960c1
20:25:57.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.503224 28379 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: loading cluster ID for follower catalog manager: success
20:25:57.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.512185 28379 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: acquiring CA information for follower catalog manager: success
20:25:57.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.518806 28379 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d2b7323b84d84dbb86a46e1aca201581: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:25:57.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:57.548558 28395 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:25:57.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:57.549953 28395 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.88.129
20:25:57.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.599370 28395 tablet_server_runner.cc:78] Tablet server non-default flags:
20:25:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:25:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:25:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:25:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:25:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/data
20:25:57.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/wal
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.129:0
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/data/info.pb
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.129
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.88.129
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/logs
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:25:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:25:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:25:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:25:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:25:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:25:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:25:57.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.600960 28395 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:25:57.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.602990 28395 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:25:57.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:57.627391 28434 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:57.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.766821 28443 sys_catalog.cc:455] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "85064d43662b4271b17b747916fe3ba7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d2b7323b84d84dbb86a46e1aca201581" member_type: VOTER last_known_addr { host: "127.27.88.190" port: 38777 } } peers { permanent_uuid: "03012590d64e4889a32c303cf20e8db4" member_type: VOTER last_known_addr { host: "127.27.88.189" port: 37063 } } peers { permanent_uuid: "85064d43662b4271b17b747916fe3ba7" member_type: VOTER last_known_addr { host: "127.27.88.188" port: 46035 } } }
20:25:57.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:57.771788 28443 sys_catalog.cc:458] T 00000000000000000000000000000000 P 85064d43662b4271b17b747916fe3ba7 [sys.catalog]: This master's current role is: LEADER
20:25:58.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:58.277043 28422 catalog_manager.cc:1259] Loaded cluster ID: c7e35025abdc48d3be8d21fa12b960c1
20:25:58.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:58.287688 28422 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4: loading cluster ID for follower catalog manager: success
20:25:58.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:58.320555 28422 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4: acquiring CA information for follower catalog manager: success
20:25:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:58.349859 28422 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 03012590d64e4889a32c303cf20e8db4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:25:59.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:59.019395 28432 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 28395
20:25:59.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:59.441203 28395 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.817s user 0.374s sys 0.819s
20:25:59.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:59.441561 28395 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.817s user 0.374s sys 0.819s
20:25:59.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:57.628583 28433 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:59.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:59.443907 28436 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:25:59.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:25:59.447116 28435 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:25:59.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.447204 28395 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:25:59.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.448985 28395 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:25:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.464282 28395 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:25:59.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.466099 28395 hybrid_clock.cc:648] HybridClock initialized: now 1786047959466050 us; error 56 us; skew 500 ppm
20:25:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.467259 28395 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:25:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.496407 28395 webserver.cc:460] Webserver started at http://127.27.88.129:34043/ using document root <none> and password file <none>
20:25:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.498136 28395 fs_manager.cc:359] Metadata directory not provided
20:25:59.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.498700 28395 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:25:59.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.499446 28395 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:25:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.512323 28395 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/data/instance:
20:25:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85f0918685da49c6b63e9c8bca7cbf54"
20:25:59.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:59 on dist-test-slave-jpj1"
20:25:59.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.516193 28395 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/wal/instance:
20:25:59.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85f0918685da49c6b63e9c8bca7cbf54"
20:25:59.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:59 on dist-test-slave-jpj1"
20:25:59.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.555648 28395 fs_manager.cc:691] Time spent creating directory manager: real 0.037s user 0.018s sys 0.016s
20:25:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.576562 28453 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:25:59.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.579087 28395 fs_manager.cc:725] Time spent opening block manager: real 0.013s user 0.006s sys 0.003s
20:25:59.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.579873 28395 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/wal
20:25:59.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "85f0918685da49c6b63e9c8bca7cbf54"
20:25:59.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:25:59 on dist-test-slave-jpj1"
20:25:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.580708 28395 fs_report.cc:383] FS layout report
20:25:59.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/wal
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/wal
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/data/data
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:25:59.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:25:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:25:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:25:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:25:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:25:59.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:25:59.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.748096 28395 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:25:59.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.752357 28395 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:25:59.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.753706 28395 kserver.cc:163] Server-wide thread pool size limit: 3276
20:25:59.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.760454 28395 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:25:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.781919 28395 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:25:59.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.782490 28395 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
20:25:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.783041 28395 ts_tablet_manager.cc:610] Registered 0 tablets
20:25:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:25:59.783450 28395 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:26:00.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.184214 28395 rpc_server.cc:307] RPC server started. Bound to: 127.27.88.129:39379
20:26:00.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.185592 28626 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.88.129:39379 every 8 connection(s)
20:26:00.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.191294 28395 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/data/info.pb
20:26:00.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.196762 28002 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 28395
20:26:00.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.197573 28002 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-0/wal/instance
20:26:00.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.261194 28002 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:00.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/wal
20:26:00.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/data
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/logs
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/data/info.pb
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:00.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:26:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:26:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:26:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:26:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:00.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:00.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:26:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:26:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.130:0
20:26:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.88.130
20:26:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.130
20:26:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:26:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:26:00.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:00.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:00.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:26:00.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.405602 28628 heartbeater.cc:347] Connected to a master server at 127.27.88.189:37063
20:26:00.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.406778 28628 heartbeater.cc:464] Registering TS with master...
20:26:00.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.411868 28627 heartbeater.cc:347] Connected to a master server at 127.27.88.188:46035
20:26:00.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.412672 28627 heartbeater.cc:464] Registering TS with master...
20:26:00.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.415657 28628 heartbeater.cc:511] Master 127.27.88.189:37063 requested a full tablet report, sending...
20:26:00.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.422870 28170 ts_manager.cc:194] Registered new tserver with Master: 85f0918685da49c6b63e9c8bca7cbf54 (127.27.88.129:39379)
20:26:00.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.424428 28627 heartbeater.cc:511] Master 127.27.88.188:46035 requested a full tablet report, sending...
20:26:00.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.433790 28276 ts_manager.cc:194] Registered new tserver with Master: 85f0918685da49c6b63e9c8bca7cbf54 (127.27.88.129:39379)
20:26:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.435315 28631 heartbeater.cc:347] Connected to a master server at 127.27.88.190:38777
20:26:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.435983 28631 heartbeater.cc:464] Registering TS with master...
20:26:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.437541 28631 heartbeater.cc:511] Master 127.27.88.190:38777 requested a full tablet report, sending...
20:26:00.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.443799 28060 ts_manager.cc:194] Registered new tserver with Master: 85f0918685da49c6b63e9c8bca7cbf54 (127.27.88.129:39379)
20:26:00.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:00.446772 28276 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.88.129:50853
20:26:01.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:01.030577 28638 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:26:01.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:01.031613 28638 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:26:01.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:01.032604 28638 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:26:01.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:01.087121 28638 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:26:01.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:01.088798 28638 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.88.130
20:26:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:01.143621 28638 tablet_server_runner.cc:78] Tablet server non-default flags:
20:26:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/data
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/wal
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.130:0
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/data/info.pb
20:26:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.130
20:26:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:26:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:26:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:26:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:26:01.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.88.130
20:26:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/logs
20:26:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:26:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:26:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:26:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:26:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:26:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:26:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:26:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:26:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:26:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:01.146098 28638 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:26:01.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:01.150336 28638 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:26:01.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:01.268152 28650 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:01.457381 28627 heartbeater.cc:503] Master 127.27.88.188:46035 was elected leader, sending a full tablet report...
20:26:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:02.623359 28649 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 28638
20:26:02.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:02.965427 28638 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.716s user 0.687s sys 0.971s
20:26:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:02.965826 28638 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.717s user 0.687s sys 0.971s
20:26:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:01.272295 28651 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:02.966432 28653 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:26:02.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:02.967643 28657 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:02.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.967698 28638 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:26:02.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.971298 28638 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:26:02.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.973516 28638 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:26:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.974889 28638 hybrid_clock.cc:648] HybridClock initialized: now 1786047962974850 us; error 40 us; skew 500 ppm
20:26:02.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.975747 28638 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:26:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.981889 28638 webserver.cc:460] Webserver started at http://127.27.88.130:36065/ using document root <none> and password file <none>
20:26:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.982870 28638 fs_manager.cc:359] Metadata directory not provided
20:26:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.983089 28638 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:26:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.983633 28638 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:26:02.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.988659 28638 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/data/instance:
20:26:02.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65495442c5c940bda943439d31c2b5fb"
20:26:02.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:02 on dist-test-slave-jpj1"
20:26:02.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:02.989897 28638 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/wal/instance:
20:26:02.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65495442c5c940bda943439d31c2b5fb"
20:26:02.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:02 on dist-test-slave-jpj1"
20:26:03.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.000382 28638 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.011s sys 0.000s
20:26:03.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.009095 28669 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:26:03.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.010357 28638 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s
20:26:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.010771 28638 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/wal
20:26:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65495442c5c940bda943439d31c2b5fb"
20:26:03.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:02 on dist-test-slave-jpj1"
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.011119 28638 fs_report.cc:383] FS layout report
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/wal
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/wal
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/data/data
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:26:03.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:26:03.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.058976 28638 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:26:03.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.060827 28638 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:26:03.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.061327 28638 kserver.cc:163] Server-wide thread pool size limit: 3276
20:26:03.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.064054 28638 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:26:03.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.068500 28638 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:26:03.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.068722 28638 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:26:03.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.069041 28638 ts_tablet_manager.cc:610] Registered 0 tablets
20:26:03.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.069206 28638 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:26:03.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.234571 28638 rpc_server.cc:307] RPC server started. Bound to: 127.27.88.130:44939
20:26:03.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.234918 28834 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.88.130:44939 every 8 connection(s)
20:26:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.237622 28638 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/data/info.pb
20:26:03.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.246475 28002 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 28638
20:26:03.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.247254 28002 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-1/wal/instance
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.272945 28002 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/wal
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/data
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:03.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/logs
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/data/info.pb
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.131:0
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.88.131
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.131
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:03.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:26:03.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.302799 28841 heartbeater.cc:347] Connected to a master server at 127.27.88.190:38777
20:26:03.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.303229 28841 heartbeater.cc:464] Registering TS with master...
20:26:03.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.304534 28841 heartbeater.cc:511] Master 127.27.88.190:38777 requested a full tablet report, sending...
20:26:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.307314 28837 heartbeater.cc:347] Connected to a master server at 127.27.88.189:37063
20:26:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.307698 28837 heartbeater.cc:464] Registering TS with master...
20:26:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.307669 28060 ts_manager.cc:194] Registered new tserver with Master: 65495442c5c940bda943439d31c2b5fb (127.27.88.130:44939)
20:26:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.308075 28835 heartbeater.cc:347] Connected to a master server at 127.27.88.188:46035
20:26:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.308415 28835 heartbeater.cc:464] Registering TS with master...
20:26:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.308540 28837 heartbeater.cc:511] Master 127.27.88.189:37063 requested a full tablet report, sending...
20:26:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.309216 28835 heartbeater.cc:511] Master 127.27.88.188:46035 requested a full tablet report, sending...
20:26:03.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.311136 28276 ts_manager.cc:194] Registered new tserver with Master: 65495442c5c940bda943439d31c2b5fb (127.27.88.130:44939)
20:26:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.311638 28170 ts_manager.cc:194] Registered new tserver with Master: 65495442c5c940bda943439d31c2b5fb (127.27.88.130:44939)
20:26:03.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.312871 28276 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.88.130:37105
20:26:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:03.619001 28850 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:26:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:03.619503 28850 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:26:03.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:03.619993 28850 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:26:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:03.651785 28850 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:26:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:03.652606 28850 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.88.131
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.687701 28850 tablet_server_runner.cc:78] Tablet server non-default flags:
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.88.148:44671
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/data
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/wal
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.88.131:0
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/data/info.pb
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.88.131
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.88.190:38777,127.27.88.189:37063,127.27.88.188:46035
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.88.131
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/logs
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:26:03.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:26:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:26:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:26:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:26:03.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.689029 28850 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:26:03.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:03.690843 28850 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:26:03.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:03.711354 28856 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:04.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.316720 28835 heartbeater.cc:503] Master 127.27.88.188:46035 was elected leader, sending a full tablet report...
20:26:04.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:03.711359 28857 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:04.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:03.711519 28859 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:04.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:04.824092 28858 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:26:04.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.824230 28850 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:26:04.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.829030 28850 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:26:04.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.831776 28850 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:26:04.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.833441 28850 hybrid_clock.cc:648] HybridClock initialized: now 1786047964833398 us; error 38 us; skew 500 ppm
20:26:04.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.834492 28850 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:26:04.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.843325 28850 webserver.cc:460] Webserver started at http://127.27.88.131:43765/ using document root <none> and password file <none>
20:26:04.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.844499 28850 fs_manager.cc:359] Metadata directory not provided
20:26:04.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.844755 28850 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:26:04.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.845278 28850 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:26:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.852113 28850 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/data/instance:
20:26:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4345b7b40b9a423788649a28c846e7e7"
20:26:04.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:04 on dist-test-slave-jpj1"
20:26:04.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.853826 28850 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/wal/instance:
20:26:04.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4345b7b40b9a423788649a28c846e7e7"
20:26:04.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:04 on dist-test-slave-jpj1"
20:26:04.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.861758 28850 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
20:26:04.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.867425 28868 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:26:04.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.868537 28850 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
20:26:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.868880 28850 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/wal
20:26:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4345b7b40b9a423788649a28c846e7e7"
20:26:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:04 on dist-test-slave-jpj1"
20:26:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.869217 28850 fs_report.cc:383] FS layout report
20:26:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:26:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/wal
20:26:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/wal
20:26:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/data/data
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:26:04.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:26:04.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.906137 28850 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:26:04.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.907781 28850 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:26:04.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.908284 28850 kserver.cc:163] Server-wide thread pool size limit: 3276
20:26:04.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.910894 28850 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
20:26:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.915126 28850 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:26:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.915331 28850 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:26:04.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.915663 28850 ts_tablet_manager.cc:610] Registered 0 tablets
20:26:04.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:04.915825 28850 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:26:05.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.092378 28850 rpc_server.cc:307] RPC server started. Bound to: 127.27.88.131:40343
20:26:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.092512 29022 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.88.131:40343 every 8 connection(s)
20:26:05.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.095381 28850 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/data/info.pb
20:26:05.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.099944 28002 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 28850
20:26:05.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.100409 28002 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster7810556852802744541/ts-2/wal/instance
20:26:05.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.142333 29025 heartbeater.cc:347] Connected to a master server at 127.27.88.189:37063
20:26:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.142844 29025 heartbeater.cc:464] Registering TS with master...
20:26:05.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.144327 29025 heartbeater.cc:511] Master 127.27.88.189:37063 requested a full tablet report, sending...
20:26:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.147027 29024 heartbeater.cc:347] Connected to a master server at 127.27.88.188:46035
20:26:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.147145 29027 heartbeater.cc:347] Connected to a master server at 127.27.88.190:38777
20:26:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.147526 29027 heartbeater.cc:464] Registering TS with master...
20:26:05.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.147573 29024 heartbeater.cc:464] Registering TS with master...
20:26:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.147719 28170 ts_manager.cc:194] Registered new tserver with Master: 4345b7b40b9a423788649a28c846e7e7 (127.27.88.131:40343)
20:26:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.148871 29027 heartbeater.cc:511] Master 127.27.88.190:38777 requested a full tablet report, sending...
20:26:05.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.149246 29024 heartbeater.cc:511] Master 127.27.88.188:46035 requested a full tablet report, sending...
20:26:05.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.152103 28060 ts_manager.cc:194] Registered new tserver with Master: 4345b7b40b9a423788649a28c846e7e7 (127.27.88.131:40343)
20:26:05.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.152364 28276 ts_manager.cc:194] Registered new tserver with Master: 4345b7b40b9a423788649a28c846e7e7 (127.27.88.131:40343)
20:26:05.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.154969 28276 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.88.131:60079
20:26:05.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:05.170594 28002 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
20:26:05.186 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:26:05.187 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:26:05.198 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.88.190"
port: 38777
}
bound_http_address {
host: "127.27.88.190"
port: 32883
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.88.189"
port: 37063
}
bound_http_address {
host: "127.27.88.189"
port: 36317
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.88.188"
port: 46035
}
bound_http_address {
host: "127.27.88.188"
port: 38439
}
}
}
20:26:05.205 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:26:05.208 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.88.129"
port: 39379
}
bound_http_address {
host: "127.27.88.129"
port: 34043
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.88.130"
port: 44939
}
bound_http_address {
host: "127.27.88.130"
port: 36065
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.88.131"
port: 40343
}
bound_http_address {
host: "127.27.88.131"
port: 43765
}
}
}
20:26:05.209 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:26:05.466 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:26:05.639 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskIlcT6h/test-tmp
20:26:05.640 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:26:05.640 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskIlcT6h/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:26:05.643 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:26:05.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:05.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:26:06.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:06.001212 29045 mini_oidc.cc:152] Starting JWKS server
20:26:06.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:06.007360 29045 webserver.cc:460] Webserver started at http://127.0.0.1:33453/ using document root <none> and password file <none>
20:26:06.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:06.008013 29045 mini_oidc.cc:189] Starting OIDC Discovery server
20:26:06.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:06.010671 29045 webserver.cc:460] Webserver started at http://127.0.0.1:37097/ using document root <none> and password file <none>
20:26:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:26:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/principal' for realm 'KRBTEST.COM',
20:26:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:26:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:06 dist-test-slave-jpj1 krb5kdc[29067](info): setting up network...
20:26:06.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:26:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:06 dist-test-slave-jpj1 krb5kdc[29067](info): set up 2 sockets
20:26:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:06 dist-test-slave-jpj1 krb5kdc[29067](info): commencing operation
20:26:06.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:26:06.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:06.160056 29024 heartbeater.cc:503] Master 127.27.88.188:46035 was elected leader, sending a full tablet report...
20:26:07.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.253293 29045 mini_kdc.cc:121] Time spent starting KDC: real 1.214s user 0.000s sys 0.027s
20:26:07.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:26:07.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:26:07.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:26:07.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:26:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:26:07.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:26:07.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:26:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:26:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:26:07.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:26:07.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/test-user.keytab.
20:26:07.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/test-user.keytab.
20:26:07.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:07 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047967, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:07.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:26:07.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:26:07Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
20:26:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:26:07Z Disabled control of system clock
20:26:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.93.126@KRBTEST.COM; defaulting to no policy
20:26:07.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:26:07.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.93.126@KRBTEST.COM" created.
20:26:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:26:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.93.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.126.keytab.
20:26:07.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.93.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.126.keytab.
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:07.468824 29045 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/logs
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/info.pb
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:26:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.126:37603
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.126
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.126.keytab
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.126
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:26:07.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:07.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:26:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.801600 29089 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:26:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.802173 29089 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:26:07.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.802749 29089 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:26:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.833913 29089 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:26:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.834193 29089 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:26:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.834507 29089 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:26:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.834735 29089 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:26:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.834918 29089 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:26:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.835219 29089 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:26:07.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.835391 29089 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:07.871845 29089 master_runner.cc:386] Master server non-default flags:
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.126.keytab
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.126
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.126:37603
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/info.pb
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.126
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/logs
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:26:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:07.873075 29089 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:26:07.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:07.874063 29089 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:26:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.898903 29098 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.900244 29097 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:09.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:07.899263 29102 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.058775 29099 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1160 milliseconds
20:26:09.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.058916 29089 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:26:09.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.060307 29089 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:26:09.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.063022 29089 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:26:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.064409 29089 hybrid_clock.cc:648] HybridClock initialized: now 1786047969064369 us; error 125 us; skew 500 ppm
20:26:09.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.065217 29089 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:26:09.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:09 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047969, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:09.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.076505 29089 init.cc:377] Logged in from keytab as kudu/127.28.93.126@KRBTEST.COM (short username kudu)
20:26:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.081694 29089 webserver.cc:460] Webserver started at http://127.28.93.126:45961/ using document root <none> and password file <none>
20:26:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.082619 29089 fs_manager.cc:359] Metadata directory not provided
20:26:09.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.082821 29089 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:26:09.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.083217 29089 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:26:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.088330 29089 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/instance:
20:26:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6a6df4b75684de89d02e0cc31f2a9b3"
20:26:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:09 on dist-test-slave-jpj1"
20:26:09.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.089672 29089 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal/instance:
20:26:09.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6a6df4b75684de89d02e0cc31f2a9b3"
20:26:09.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:09 on dist-test-slave-jpj1"
20:26:09.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.100819 29089 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.008s sys 0.004s
20:26:09.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.108336 29110 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:26:09.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.109422 29089 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.109779 29089 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6a6df4b75684de89d02e0cc31f2a9b3"
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:09 on dist-test-slave-jpj1"
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.110122 29089 fs_report.cc:383] FS layout report
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/data
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:26:09.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:26:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:26:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:26:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:26:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:26:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:26:09.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:26:09.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.154811 29089 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:26:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.160723 29089 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:26:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.161232 29089 kserver.cc:163] Server-wide thread pool size limit: 3276
20:26:09.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.284900 29089 rpc_server.cc:307] RPC server started. Bound to: 127.28.93.126:37603
20:26:09.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.285010 29188 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.93.126:37603 every 8 connection(s)
20:26:09.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.288138 29089 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/info.pb
20:26:09.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.294914 29045 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 29089
20:26:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.295437 29045 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal/instance
20:26:09.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.303300 29189 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:26:09.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.93.125@KRBTEST.COM; defaulting to no policy
20:26:09.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.323405 29189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } has no permanent_uuid. Determining permanent_uuid...
20:26:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:26:09.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.93.125@KRBTEST.COM" created.
20:26:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:09 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047969, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.126@KRBTEST.COM for kudu/127.28.93.126@KRBTEST.COM
20:26:09.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:26:09.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.93.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.125.keytab.
20:26:09.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.93.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.125.keytab.
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.365945 29045 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/logs
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/info.pb
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:09.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.125:36801
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.125
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.125.keytab
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.125
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:09.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:26:09.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.377223 29189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } has no permanent_uuid. Determining permanent_uuid...
20:26:09.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.379480 29116 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:09.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.383811 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.125:36801: Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:09.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.433456 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } attempt: 1
20:26:09.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.437119 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.125:36801: Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:09.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.505555 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } attempt: 2
20:26:09.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.509049 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.125:36801: Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.600510 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } attempt: 3
20:26:09.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.604152 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.125:36801: Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.716226 29196 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:26:09.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.716764 29196 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:26:09.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.717347 29196 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:26:09.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.747627 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } attempt: 4
20:26:09.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.749042 29196 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:26:09.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.749254 29196 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:26:09.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.749509 29196 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:26:09.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.749804 29196 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:26:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.750104 29196 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:26:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.750396 29196 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:26:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.750607 29196 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
20:26:09.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.751344 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.125:36801: Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.787925 29196 master_runner.cc:386] Master server non-default flags:
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.125.keytab
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.125
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.125:36801
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/info.pb
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.125
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:26:09.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/logs
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.789178 29196 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:26:09.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:09.790251 29196 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:26:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.807859 29207 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:10.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.050805 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } attempt: 5
20:26:10.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:10.055177 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.125:36801: Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:10.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.602730 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } attempt: 6
20:26:10.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:10.610592 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.125:36801: Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.807934 29209 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:10.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:09.808859 29206 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:10.959553 29208 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:26:10.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.960019 29196 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:26:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.963976 29196 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:26:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.966948 29196 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:26:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.968535 29196 hybrid_clock.cc:648] HybridClock initialized: now 1786047970968451 us; error 93 us; skew 500 ppm
20:26:10.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.969709 29196 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:26:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:10 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047970, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:10.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.975854 29196 init.cc:377] Logged in from keytab as kudu/127.28.93.125@KRBTEST.COM (short username kudu)
20:26:10.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.982483 29196 webserver.cc:460] Webserver started at http://127.28.93.125:33633/ using document root <none> and password file <none>
20:26:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.983400 29196 fs_manager.cc:359] Metadata directory not provided
20:26:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.983635 29196 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:26:10.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.984149 29196 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:26:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.989341 29196 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/instance:
20:26:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87059507e8c449f08835ab023307bf97"
20:26:10.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:10 on dist-test-slave-jpj1"
20:26:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.990425 29196 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal/instance:
20:26:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87059507e8c449f08835ab023307bf97"
20:26:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:10 on dist-test-slave-jpj1"
20:26:10.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:10.997718 29196 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
20:26:11.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.003613 29230 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:26:11.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.004642 29196 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
20:26:11.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.004966 29196 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:11.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87059507e8c449f08835ab023307bf97"
20:26:11.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:10 on dist-test-slave-jpj1"
20:26:11.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.005373 29196 fs_report.cc:383] FS layout report
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/data
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:26:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:26:11.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.059417 29196 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:26:11.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.065443 29196 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:26:11.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.065958 29196 kserver.cc:163] Server-wide thread pool size limit: 3276
20:26:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.161336 29196 rpc_server.cc:307] RPC server started. Bound to: 127.28.93.125:36801
20:26:11.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.161430 29306 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.93.125:36801 every 8 connection(s)
20:26:11.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.164040 29196 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/info.pb
20:26:11.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.169983 29307 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:26:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.172921 29045 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 29196
20:26:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.173364 29045 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal/instance
20:26:11.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.185030 29307 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } has no permanent_uuid. Determining permanent_uuid...
20:26:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.93.124@KRBTEST.COM; defaulting to no policy
20:26:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:26:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.93.124@KRBTEST.COM" created.
20:26:11.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:11 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047970, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.125@KRBTEST.COM for kudu/127.28.93.126@KRBTEST.COM
20:26:11.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.226747 29307 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } has no permanent_uuid. Determining permanent_uuid...
20:26:11.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:26:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.93.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.124.keytab.
20:26:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.93.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.124.keytab.
20:26:11.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.235384 29045 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/logs
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/info.pb
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.124:39973
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.124
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.124.keytab
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.124
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:11.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:11.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:26:11.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:11 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047970, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.125@KRBTEST.COM for kudu/127.28.93.125@KRBTEST.COM
20:26:11.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.258463 29307 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } has no permanent_uuid. Determining permanent_uuid...
20:26:11.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.260795 29234 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.265184 29307 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.314730 29307 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 1
20:26:11.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.318173 29307 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.386631 29307 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 2
20:26:11.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.392758 29307 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.484176 29307 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 3
20:26:11.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.488142 29307 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.595104 29317 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:26:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.595718 29317 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:26:11.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.596308 29317 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:26:11.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.628661 29317 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:26:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.628890 29317 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:26:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.629277 29317 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:26:11.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.629563 29317 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:26:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.629791 29317 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:26:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.630044 29317 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:26:11.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.630285 29317 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
20:26:11.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.631515 29307 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 4
20:26:11.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.635280 29307 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.668409 29317 master_runner.cc:386] Master server non-default flags:
20:26:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:11.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.124.keytab
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.124
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.124:39973
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/info.pb
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.124
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/logs
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:26:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.669689 29317 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:26:11.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.670725 29317 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:26:11.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.671236 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } attempt: 7
20:26:11.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:11 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047969, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.126@KRBTEST.COM for kudu/127.28.93.125@KRBTEST.COM
20:26:11.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.693598 29337 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:11.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.700389 29189 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } has no permanent_uuid. Determining permanent_uuid...
20:26:11.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.706918 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.765326 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 1
20:26:11.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.773974 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.855458 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 2
20:26:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.860885 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.934738 29307 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 5
20:26:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.939365 29307 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:11.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:11.946447 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 3
20:26:11.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.950994 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.091485 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 4
20:26:12.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:12.097189 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:12.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.380667 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 5
20:26:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:12.387130 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:12.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.486872 29307 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 6
20:26:12.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:12.495160 29307 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:12.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.939752 29189 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 6
20:26:12.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:12.946669 29189 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.28.93.124:39973: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: connect: Connection refused (error 111)
20:26:12.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.697407 29332 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:12.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:11.693655 29335 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:12.969158 29336 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1274 milliseconds
20:26:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.969304 29317 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:26:12.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.970628 29317 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:26:12.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.973306 29317 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:26:12.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.974654 29317 hybrid_clock.cc:648] HybridClock initialized: now 1786047972974616 us; error 46 us; skew 500 ppm
20:26:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.975569 29317 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:26:12.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:12 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047972, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.981633 29317 init.cc:377] Logged in from keytab as kudu/127.28.93.124@KRBTEST.COM (short username kudu)
20:26:12.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.987716 29317 webserver.cc:460] Webserver started at http://127.28.93.124:46459/ using document root <none> and password file <none>
20:26:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.988770 29317 fs_manager.cc:359] Metadata directory not provided
20:26:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.989028 29317 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:26:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.989532 29317 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
20:26:12.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.994431 29317 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/instance:
20:26:12.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2969334c48144595baf8d4ee79580383"
20:26:12.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:12 on dist-test-slave-jpj1"
20:26:12.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:12.995743 29317 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal/instance:
20:26:12.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2969334c48144595baf8d4ee79580383"
20:26:12.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:12 on dist-test-slave-jpj1"
20:26:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.003252 29317 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:26:13.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.008967 29356 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:26:13.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.009999 29317 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
20:26:13.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.010360 29317 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:13.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2969334c48144595baf8d4ee79580383"
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:12 on dist-test-slave-jpj1"
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.010768 29317 fs_report.cc:383] FS layout report
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/data
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:26:13.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:26:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:26:13.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:26:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.060118 29317 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:26:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.065780 29317 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:26:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.066298 29317 kserver.cc:163] Server-wide thread pool size limit: 3276
20:26:13.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.179046 29317 rpc_server.cc:307] RPC server started. Bound to: 127.28.93.124:39973
20:26:13.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.179150 29428 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.93.124:39973 every 8 connection(s)
20:26:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.184108 29317 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/info.pb
20:26:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.190806 29429 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:26:13.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.191241 29045 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 29317
20:26:13.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.191764 29045 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal/instance
20:26:13.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.204578 29429 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } has no permanent_uuid. Determining permanent_uuid...
20:26:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:13 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047972, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.124@KRBTEST.COM for kudu/127.28.93.126@KRBTEST.COM
20:26:13.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:13 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047967, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.93.124@KRBTEST.COM
20:26:13.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.245083 29429 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } has no permanent_uuid. Determining permanent_uuid...
20:26:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:13 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047972, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.124@KRBTEST.COM for kudu/127.28.93.125@KRBTEST.COM
20:26:13.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:13 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047967, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.93.125@KRBTEST.COM
20:26:13.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.270179 29429 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } has no permanent_uuid. Determining permanent_uuid...
20:26:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:13 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047972, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.124@KRBTEST.COM for kudu/127.28.93.124@KRBTEST.COM
20:26:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:13 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047967, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.93.126@KRBTEST.COM
20:26:13.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.304594 29429 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: Bootstrap starting.
20:26:13.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.311095 29429 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: Neither blocks nor log segments found. Creating new log.
20:26:13.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.313252 29429 log.cc:824] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: Log is configured to *not* fsync() on all Append() calls
20:26:13.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.318212 29045 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
20:26:13.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.323036 29429 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: No bootstrap required, opened a new log
20:26:13.328 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:13.330 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:26:13.340 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:26:13.341 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:26:13.344 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.93.126"
port: 37603
}
bound_http_address {
host: "127.28.93.126"
port: 45961
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.93.125"
port: 36801
}
bound_http_address {
host: "127.28.93.125"
port: 33633
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.93.124"
port: 39973
}
bound_http_address {
host: "127.28.93.124"
port: 46459
}
}
}
20:26:13.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:26:13.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:13.345 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:26:13.349 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:26:13.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.359905 29429 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:13.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.360811 29429 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:26:13.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.361117 29429 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2969334c48144595baf8d4ee79580383, State: Initialized, Role: FOLLOWER
20:26:13.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.362041 29429 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.365278 29440 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } } }
20:26:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.366014 29440 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [sys.catalog]: This master's current role is: FOLLOWER
20:26:13.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.367148 29429 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [sys.catalog]: configured and running, proceeding with master startup.
20:26:13.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.390455 29440 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:13.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.391026 29440 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.393939 29440 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:13.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:13 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047973, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:13.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:26:13.409 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:13.411 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc
20:26:13.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.433079 29458 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:13.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.433806 29458 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:13.439 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:26:13.441 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc
20:26:13.443 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:26:13.448 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
20:26:13.450 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:26:13.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:26:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.419839 29317 master_runner.cc:186] Error getting master registration for 127.28.93.126:37603: OK, instance_id { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" instance_seqno: 1786047969243922 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:26:13.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.469318 29317 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:26:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:13 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047973, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:26:13.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:13.509 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc
20:26:13.513 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:26:13.513 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.93.126:37603
20:26:13.515 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:26:13.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.516408 29045 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 29089
20:26:13.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.555604 29307 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } attempt: 7
20:26:13.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.556928 29360 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.28.93.126:37603 (error 108)
20:26:13.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.561190 29360 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:13.566 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:13.566 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.93.124:39973
20:26:13.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:26:13.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.567698 29045 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 29317
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.597780 29463 negotiation.cc:337] Failed RPC negotiation. Trace:
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.556980 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.28.93.124:39973 (local address 127.0.0.1:52576)
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.565089 (+ 8109us) negotiation.cc:107] Waiting for socket to connect
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.565146 (+ 57us) client_negotiation.cc:174] Beginning negotiation
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.565405 (+ 259us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.566576 (+ 1171us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.566605 (+ 29us) client_negotiation.cc:394] Received NEGOTIATE response from server
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.570396 (+ 3791us) client_negotiation.cc:189] Negotiated authn=SASL
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.571058 (+ 662us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.571111 (+ 53us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0806 20:26:13.596242 (+ 25131us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.28.93.124:39973: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:26:13.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":7710,"thread_start_us":7447,"threads_started":1}
20:26:13.604 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:13.604 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.93.125:36801
20:26:13.604 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:26:13.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.605134 29045 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 29196
20:26:13.627 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:13.628 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.93.126:37603
20:26:13.629 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:13.631166 29045 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/logs
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/info.pb
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:26:13.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.126:37603
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.126
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45961
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.126.keytab
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.126
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:13.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:26:13.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.936125 29469 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:26:13.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.936844 29469 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:26:13.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.937557 29469 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:26:13.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.969102 29469 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:26:13.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.969370 29469 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:26:13.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.969789 29469 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:26:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.970096 29469 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:26:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.970389 29469 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:26:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.970676 29469 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:26:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:13.970896 29469 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:14.007862 29469 master_runner.cc:386] Master server non-default flags:
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.126.keytab
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.126
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.126:37603
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:14.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:26:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:26:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:26:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/info.pb
20:26:14.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.126
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45961
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/logs
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:14.009377 29469 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:26:14.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:14.010736 29469 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:26:14.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:14.028461 29478 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:14.028601 29477 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:15.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:14.029904 29469 server_base.cc:1034] running on GCE node
20:26:15.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:14.028474 29480 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.154335 29469 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:26:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.156919 29469 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:26:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.158273 29469 hybrid_clock.cc:648] HybridClock initialized: now 1786047975158229 us; error 53 us; skew 500 ppm
20:26:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.159134 29469 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:26:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:15 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047975, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.168663 29469 init.cc:377] Logged in from keytab as kudu/127.28.93.126@KRBTEST.COM (short username kudu)
20:26:15.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.174069 29469 webserver.cc:460] Webserver started at http://127.28.93.126:45961/ using document root <none> and password file <none>
20:26:15.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.175273 29469 fs_manager.cc:359] Metadata directory not provided
20:26:15.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.175591 29469 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:26:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.186772 29469 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.005s sys 0.000s
20:26:15.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.191906 29495 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:26:15.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.192998 29469 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.193352 29469 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6a6df4b75684de89d02e0cc31f2a9b3"
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:09 on dist-test-slave-jpj1"
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.196077 29469 fs_report.cc:383] FS layout report
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/data
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:26:15.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:26:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:26:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:26:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:26:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:26:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:26:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:26:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:26:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:26:15.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.230011 29469 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:26:15.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.236057 29469 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:26:15.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.236562 29469 kserver.cc:163] Server-wide thread pool size limit: 3276
20:26:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.310209 29469 rpc_server.cc:307] RPC server started. Bound to: 127.28.93.126:37603
20:26:15.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.310286 29568 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.93.126:37603 every 8 connection(s)
20:26:15.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.314062 29469 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/data/info.pb
20:26:15.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.319319 29045 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 29469
20:26:15.320 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:15.320 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.93.124:39973
20:26:15.320 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:26:15.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.321542 29045 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/logs
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/info.pb
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.124:39973
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.124
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46459
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.124.keytab
20:26:15.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.124
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:15.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:26:15.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.327003 29569 sys_catalog.cc:263] Verifying existing consensus state
20:26:15.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 20:26:15.328002 29569 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:26:15.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.328511 29469 master.cc:537] Master@127.28.93.126:37603 shutting down...
20:26:15.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.392033 29469 master.cc:552] Master@127.28.93.126:37603 shutdown complete.
20:26:15.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:26:15.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.654495 29571 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:26:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.655028 29571 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:26:15.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.655663 29571 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:26:15.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.686403 29571 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:26:15.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.686647 29571 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:26:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.686904 29571 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:26:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.687098 29571 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:26:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.687327 29571 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:26:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.687520 29571 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:26:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.687742 29571 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.723956 29571 master_runner.cc:386] Master server non-default flags:
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.124.keytab
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.124
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.124:39973
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/info.pb
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.124
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46459
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/logs
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:26:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.725147 29571 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:26:15.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.726111 29571 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:26:15.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.743351 29581 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:16.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.743356 29585 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:16.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:15.743355 29580 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:16.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:15.745832 29571 server_base.cc:1034] running on GCE node
20:26:16.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.863879 29571 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:26:16.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.866791 29571 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:26:16.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.868196 29571 hybrid_clock.cc:648] HybridClock initialized: now 1786047976868171 us; error 41 us; skew 500 ppm
20:26:16.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.868978 29571 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:26:16.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:16 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047976, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:16.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.874773 29571 init.cc:377] Logged in from keytab as kudu/127.28.93.124@KRBTEST.COM (short username kudu)
20:26:16.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.879906 29571 webserver.cc:460] Webserver started at http://127.28.93.124:46459/ using document root <none> and password file <none>
20:26:16.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.880815 29571 fs_manager.cc:359] Metadata directory not provided
20:26:16.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.881026 29571 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:26:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.889137 29571 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.004s sys 0.000s
20:26:16.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.893448 29598 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:26:16.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.894475 29571 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.894755 29571 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2969334c48144595baf8d4ee79580383"
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:12 on dist-test-slave-jpj1"
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.897148 29571 fs_report.cc:383] FS layout report
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/wal
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/data
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:26:16.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:26:16.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.943755 29571 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:26:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.949631 29571 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:26:16.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:16.950069 29571 kserver.cc:163] Server-wide thread pool size limit: 3276
20:26:17.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.021510 29571 rpc_server.cc:307] RPC server started. Bound to: 127.28.93.124:39973
20:26:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.021656 29667 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.93.124:39973 every 8 connection(s)
20:26:17.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.024209 29571 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-2/data/info.pb
20:26:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.029258 29045 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 29571
20:26:17.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:17.030 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.93.125:36801
20:26:17.031 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:26:17.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.032618 29045 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:17.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/logs
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/info.pb
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.125:36801
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.125
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33633
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.125.keytab
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.125
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:17.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:17.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:17.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:17.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:17.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:26:17.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.036161 29668 sys_catalog.cc:263] Verifying existing consensus state
20:26:17.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.043267 29668 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: Bootstrap starting.
20:26:17.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.079522 29668 log.cc:824] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: Log is configured to *not* fsync() on all Append() calls
20:26:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.084017 29668 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:26:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.084721 29668 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: Bootstrap complete.
20:26:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.112613 29668 raft_consensus.cc:348] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.115167 29668 raft_consensus.cc:374] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:26:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.115382 29668 raft_consensus.cc:729] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2969334c48144595baf8d4ee79580383, State: Initialized, Role: FOLLOWER
20:26:17.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.116218 29668 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:17.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.118937 29675 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } } }
20:26:17.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.120337 29668 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [sys.catalog]: configured and running, proceeding with master startup.
20:26:17.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.120404 29675 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [sys.catalog]: This master's current role is: FOLLOWER
20:26:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.151327 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.151965 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:17.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.145853 29571 master_runner.cc:186] Error getting master registration for 127.28.93.126:37603: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111),
20:26:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.174335 29675 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.174450 29571 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:26:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.174850 29675 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.178035 29675 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:17.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.187896 29603 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.199503 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801): Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.200431 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:17.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.201035 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2969334c48144595baf8d4ee79580383; no voters: 87059507e8c449f08835ab023307bf97, b6a6df4b75684de89d02e0cc31f2a9b3
20:26:17.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.202173 29675 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:26:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.205044 29607 tablet.cc:2369] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:26:17.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.427086 29673 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
20:26:17.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.428061 29673 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:26:17.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.428989 29673 flags.cc:425] Enabled unsafe flag: --never_fsync=true
20:26:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.467774 29673 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
20:26:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.468048 29673 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
20:26:17.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.468467 29673 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
20:26:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.468782 29673 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
20:26:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.469362 29673 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
20:26:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.469760 29673 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
20:26:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.470134 29673 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:33453/jwks.json
20:26:17.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.514250 29673 master_runner.cc:386] Master server non-default flags:
20:26:17.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:26:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.93.84:42389
20:26:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:26:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:26:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data
20:26:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:26:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:26:17.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.93.126:37603,127.28.93.125:36801,127.28.93.124:39973
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/krb5kdc/kudu_127.28.93.125.keytab
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.93.125
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.93.125:36801
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:33453/jwks.json
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/info.pb
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.93.125
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33633
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:26:17.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/logs
20:26:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:26:17.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 06 Aug 2026 19:03:46 UTC on ccycloud.quasar-tvywfk.root.comops.site
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4570856
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.515908 29673 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:26:17.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:17.517390 29673 file_cache.cc:493] Constructed file cache file cache with capacity 419430
20:26:17.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.548779 29707 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:18.157166 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:18.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:18.157958 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.162575 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.163266 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:18.939409 29705 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 29673
20:26:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.257694 29705 kernel_stack_watchdog.cc:198] Thread 29673 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 385ms:
20:26:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:26:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:26:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:26:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:26:19.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:17.552078 29708 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.259829 29673 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.708s user 0.607s sys 0.928s
20:26:19.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.260178 29673 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.708s user 0.607s sys 0.928s
20:26:19.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.261291 29711 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:26:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.264498 29673 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:26:19.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.264411 29710 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1711 milliseconds
20:26:19.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.266779 29673 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:26:19.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.269371 29673 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:26:19.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.270876 29673 hybrid_clock.cc:648] HybridClock initialized: now 1786047979270850 us; error 49 us; skew 500 ppm
20:26:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.271880 29673 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:26:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:19 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047979, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.277904 29673 init.cc:377] Logged in from keytab as kudu/127.28.93.125@KRBTEST.COM (short username kudu)
20:26:19.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.287948 29673 webserver.cc:460] Webserver started at http://127.28.93.125:33633/ using document root <none> and password file <none>
20:26:19.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.289256 29673 fs_manager.cc:359] Metadata directory not provided
20:26:19.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.289675 29673 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:26:19.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.300148 29673 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
20:26:19.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.305414 29725 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:26:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.306833 29673 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
20:26:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.307185 29673 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data,/tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "87059507e8c449f08835ab023307bf97"
20:26:19.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-06 20:26:10 on dist-test-slave-jpj1"
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.310397 29673 fs_report.cc:383] FS layout report
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/data
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:26:19.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:26:19.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.355451 29673 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
20:26:19.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.361151 29673 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:26:19.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.361644 29673 kserver.cc:163] Server-wide thread pool size limit: 3276
20:26:19.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.391953 29748 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.392421 29748 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:19.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.394065 29748 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.410665 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:19.411248 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801): Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:19.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.411590 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2969334c48144595baf8d4ee79580383; no voters: 87059507e8c449f08835ab023307bf97, b6a6df4b75684de89d02e0cc31f2a9b3
20:26:19.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.412325 29748 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:26:19.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.459620 29673 rpc_server.cc:307] RPC server started. Bound to: 127.28.93.125:36801
20:26:19.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.459671 29802 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.28.93.125:36801 every 8 connection(s)
20:26:19.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.463436 29673 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/data/info.pb
20:26:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.471158 29045 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu as pid 29673
20:26:19.471 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:26:19.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.474830 29803 sys_catalog.cc:263] Verifying existing consensus state
20:26:19.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260806 20:26:19.475749 29803 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:26:19.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.476246 29673 master.cc:537] Master@127.28.93.125:36801 shutting down...
20:26:19.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:19.553025 29673 master.cc:552] Master@127.28.93.125:36801 shutdown complete.
20:26:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskIlcT6h/test-tmp/mini-kudu-cluster2289326982064698983/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:26:19.567 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:19.642 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.93.124:39973(127.28.93.124:39973)] connecting to peer
20:26:19.643 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:19.673 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.93.124:39973(127.28.93.124:39973)] Successfully connected to peer
20:26:19.676 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:19.677 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:19.679 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:19.680 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:19.680 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:19.681 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:19.748 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
20:26:19.769 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.28.93.124
20:26:20.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:20 dist-test-slave-jpj1 krb5kdc[29067](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047973, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.93.124@KRBTEST.COM
20:26:20.056 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x57a928ac, L:/127.0.0.1:52588 - R:/127.28.93.124:39973] using SASL/GSSAPI
20:26:20.071 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.080 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.083 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.090 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.092 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.099 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.100 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.102 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.103 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.107 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.110 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.110 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.120 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.121 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.123 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.123 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.124 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.124 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.124 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.125 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.126 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.140 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.141 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.143 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.143 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.144 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.144 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.145 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.145 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.146 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.160 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.161 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.162 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.163 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.163 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.163 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.163 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:20.166270 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:20.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:20.166570 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:20.167 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.168 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.219 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.221 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.223 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.223 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.223 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.223 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.225 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.225 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.227 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.280 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.282 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.284 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.284 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.284 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.284 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.284 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.285 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.287 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.320 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.321 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.323 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.323 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.323 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.325 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.326 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.328 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.800 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:20.801 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:20.803 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.803 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.804 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.804 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.804 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:20.804 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:20.806 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:20.852782 28322 debug-util.cc:398] Leaking SignalData structure 0x7b08000bcc80 after lost signal to thread 28225
20:26:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:20.853358 28322 debug-util.cc:398] Leaking SignalData structure 0x7b0800081880 after lost signal to thread 28327
20:26:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:21.169400 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:21.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:21.169668 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:21.212159 29812 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:21.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:21.212591 29812 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:21.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:21.214232 29812 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:21.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:21.224251 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:21.224859 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801): Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:21.225134 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2969334c48144595baf8d4ee79580383; no voters: 87059507e8c449f08835ab023307bf97, b6a6df4b75684de89d02e0cc31f2a9b3
20:26:21.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:21.225636 29812 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:26:21.840 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:21.841 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:21.843 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:21.843 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:21.843 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:21.843 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:21.843 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:21.843 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:21.845 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:22.172515 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:22.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:22.172772 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:22.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:22.177693 29571 master_runner.cc:186] Error getting master registration for 127.28.93.126:37603: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111),
20:26:22.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:22.178112 29571 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:26:23.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:23.141520 29821 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:23.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:23.141973 29821 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:23.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:23.145803 29821 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:23.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:23.146663 29603 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:26:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:23.149825 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:23.150236 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801): Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:23.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:23.150480 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2969334c48144595baf8d4ee79580383; no voters: 87059507e8c449f08835ab023307bf97, b6a6df4b75684de89d02e0cc31f2a9b3
20:26:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:23.151034 29821 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:26:23.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:23.176476 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:23.176784 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:23.279 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:23.281 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:23.283 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:23.283 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:23.284 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:23.284 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:23.284 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:23.284 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:23.285 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:23.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:23.856144 28831 debug-util.cc:398] Leaking SignalData structure 0x7b08000af060 after lost signal to thread 28640
20:26:23.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:23.857076 28831 debug-util.cc:398] Leaking SignalData structure 0x7b08000afa20 after lost signal to thread 28834
20:26:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:24.179782 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:24.180028 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:25.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:25.183140 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:25.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:25.183431 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:25.360 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:25.361 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:25.364 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:25.364 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:25.364 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:25.364 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:25.364 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:25.364 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:25.365 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:26.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:26.187606 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:26.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:26.187948 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:26.487227 29832 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:26.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:26.487800 29832 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:26.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:26.489826 29832 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:26.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:26.495496 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:26.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:26.496074 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801): Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:26.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:26.496331 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2969334c48144595baf8d4ee79580383; no voters: 87059507e8c449f08835ab023307bf97, b6a6df4b75684de89d02e0cc31f2a9b3
20:26:26.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:26.496907 29832 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:26:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:26 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047986, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:26.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:26.878587 29592 init.cc:295] Successfully reacquired a new kerberos TGT
20:26:27.179 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:27.181 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:27.182 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:27.182 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:27.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:27.181592 29571 master_runner.cc:186] Error getting master registration for 127.28.93.126:37603: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111),
20:26:27.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:27.182087 29571 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:26:27.182 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:27.184 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:27.184 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:27.184 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:27.187 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:27.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:27.190745 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:27.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:27.190980 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:28.193848 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:28.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:28.194087 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:29.196931 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:29.197189 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:30.199930 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:30.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:30.200171 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:30.900 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:30.902 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:30.902 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:30.903 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:30.904 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:30.906 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:30.906 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:30.906 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:30.908 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:31.202831 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:31.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:31.203076 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:31.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:31.899151 29019 debug-util.cc:398] Leaking SignalData structure 0x7b08000af560 after lost signal to thread 28851
20:26:31.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:31.900133 29019 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6080 after lost signal to thread 29022
20:26:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:32.184981 29571 master_runner.cc:186] Error getting master registration for 127.28.93.126:37603: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111),
20:26:32.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:32.185334 29571 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:26:32.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:32.205860 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:32.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:32.206117 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:32.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:32.303401 29845 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:32.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:32.303798 29845 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:32.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:32.306299 29845 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:32.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:32.307082 29603 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:26:32.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:32.312121 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:32.312785 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801): Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:32.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:32.313151 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2969334c48144595baf8d4ee79580383; no voters: 87059507e8c449f08835ab023307bf97, b6a6df4b75684de89d02e0cc31f2a9b3
20:26:32.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:32.313864 29845 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:26:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:33.210065 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:33.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:33.210398 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:34.213327 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:34.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:34.213572 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:34.239 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:34.240 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:34.242 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:34.243 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:34.243 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:34.243 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:34.243 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:34.244 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:34.244 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:34.660 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:34.661 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:34.663 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:34.663 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:34.663 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:34.663 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:34.663 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:34.663 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:34.665 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:34.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:34.973038 29854 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:34.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:34.973466 29854 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:34.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:34.975008 29854 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:34.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:34.980688 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:34.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:34.981208 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801): Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:34.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:34.981469 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2969334c48144595baf8d4ee79580383; no voters: 87059507e8c449f08835ab023307bf97, b6a6df4b75684de89d02e0cc31f2a9b3
20:26:34.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:34.982080 29854 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:26:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:35.216743 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:35.217008 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:36.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:36.220585 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:36.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:36.220920 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:36.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:36 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786047996, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:36.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:36.880959 29592 init.cc:295] Successfully reacquired a new kerberos TGT
20:26:37.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:37.188534 29571 master_runner.cc:186] Error getting master registration for 127.28.93.126:37603: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111),
20:26:37.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:37.188927 29571 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:26:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:37.223742 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:37.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:37.223989 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:37.228891 28211 debug-util.cc:398] Leaking SignalData structure 0x7b080007b5a0 after lost signal to thread 28120
20:26:37.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:37.229472 28211 debug-util.cc:398] Leaking SignalData structure 0x7b08000ab5c0 after lost signal to thread 28215
20:26:37.339 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:37.342 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:37.343 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:37.344 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:37.344 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:37.344 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:37.344 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:37.345 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:37.346 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:38.227917 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:38.228253 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:39.000 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:39.001 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:39.003 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:39.003 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:39.003 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:39.004 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:39.004 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:39.005 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:39.006 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:39.231285 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:39.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:39.231597 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:39.901969 28831 debug-util.cc:398] Leaking SignalData structure 0x7b08000afb80 after lost signal to thread 28640
20:26:39.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:39.902765 28831 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5860 after lost signal to thread 28834
20:26:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:40.234520 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:40.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:40.234771 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:41.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:41.237509 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:41.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:41.237762 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:41.400 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:41.402 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:41.403 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:41.404 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:41.404 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:41.404 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:41.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:41.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:41.408 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:42.191424 29571 master_runner.cc:186] Error getting master registration for 127.28.93.126:37603: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111),
20:26:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:42.191844 29571 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:26:42.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:42.240602 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:42.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:42.240866 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:43.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:43.243635 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:43.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:43.243952 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:44.246791 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:44.247054 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:44.580 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:44.581 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:44.582 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:44.582 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:44.582 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:44.583 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:44.583 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:44.583 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:44.584 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:45.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:45.249801 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:45.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:45.250047 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:46.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:46.253587 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:46.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:46.253949 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:46.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:46.286505 29879 raft_consensus.cc:482] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:26:46.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:46.286948 29879 raft_consensus.cc:504] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b6a6df4b75684de89d02e0cc31f2a9b3" member_type: VOTER last_known_addr { host: "127.28.93.126" port: 37603 } } peers { permanent_uuid: "87059507e8c449f08835ab023307bf97" member_type: VOTER last_known_addr { host: "127.28.93.125" port: 36801 } } peers { permanent_uuid: "2969334c48144595baf8d4ee79580383" member_type: VOTER last_known_addr { host: "127.28.93.124" port: 39973 } }
20:26:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:46.288525 29879 leader_election.cc:288] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603), 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801)
20:26:46.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:46.291452 29603 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111) [suppressed 3 similar messages]
20:26:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:46.294858 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b6a6df4b75684de89d02e0cc31f2a9b3 (127.28.93.126:37603): Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111)
20:26:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:46.295449 29603 leader_election.cc:334] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 87059507e8c449f08835ab023307bf97 (127.28.93.125:36801): Network error: Client connection negotiation failed: client connection to 127.28.93.125:36801: connect: Connection refused (error 111)
20:26:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:46.295763 29603 leader_election.cc:302] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2969334c48144595baf8d4ee79580383; no voters: 87059507e8c449f08835ab023307bf97, b6a6df4b75684de89d02e0cc31f2a9b3
20:26:46.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:46.296353 29879 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:26:46.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Aug 06 20:26:46 dist-test-slave-jpj1 krb5kdc[29067](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1786048006, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.93.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:26:46.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:46.883322 29592 init.cc:295] Successfully reacquired a new kerberos TGT
20:26:47.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:47.194664 29571 master_runner.cc:186] Error getting master registration for 127.28.93.126:37603: Network error: Client connection negotiation failed: client connection to 127.28.93.126:37603: connect: Connection refused (error 111),
20:26:47.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:47.195087 29571 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
20:26:47.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:47.256664 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:47.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:47.256925 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:48.239 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:48.241 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:48.241 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:48.241 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:48.242 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:48.242 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:48.242 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:48.242 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:48.245 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:48.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:48.259191 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:48.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:48.259438 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:48.859 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:48.860 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:48.861 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:48.861 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:48.861 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:48.861 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:48.862 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:48.862 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:48.863 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:49.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:49.261901 29691 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 2969334c48144595baf8d4ee79580383: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:26:49.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260806 20:26:49.262132 29691 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:26:49.400 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.126:37603(127.28.93.126:37603)] connecting to peer
20:26:49.401 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.28.93.125:36801(127.28.93.125:36801)] connecting to peer
20:26:49.402 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:49.403 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.126:37603(127.28.93.126:37603)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:49.403 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:49.403 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.126:37603: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603
20:26:49.403 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.93.125:36801(127.28.93.125:36801)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:49.403 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.93.125:36801: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
20:26:49.407 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801; will retry
20:26:49.412 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29930), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [194ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [197ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [583ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [605ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [606ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [608ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [608ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [619ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [624ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [626ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [639ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [640ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [642ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [643ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [660ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [660ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [662ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [679ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [679ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [683ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [742ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [744ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [802ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [803ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [835ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [839ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [841ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [843ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [844ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [1315ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [1323ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [2355ms] refreshing cache from master, [2355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [2357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [2361ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [2361ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [3795ms] refreshing cache from master, [3795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [3796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [3797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [3800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [3800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [3801ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [3801ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [5875ms] refreshing cache from master, [5876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [5876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [5877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [5880ms] too many traces: truncated at 100 traces, deferred=Deferred@1665289055(state=PENDING, result=null, callback=(continuation of Deferred@663248745 after retry RPC after error@1494206213) -> (continuation of Deferred@1714759237 after retry RPC after error@391938415) -> (continuation of Deferred@1917511674 after retry RPC after error@1440362841) -> (continuation of Deferred@437396137 after retry RPC after error@1773571965) -> (continuation of Deferred@1290195078 after retry RPC after error@1219649722) -> (continuation of Deferred@1128141935 after retry RPC after error@514766354) -> (continuation of Deferred@1681274785 after retry RPC after error@1071262765) -> (continuation of Deferred@217369301 after retry RPC after error@1736569727) -> (continuation of Deferred@845105937 after retry RPC after error@1552593459) -> (continuation of Deferred@1988262165 after retry RPC after error@519075505) -> (continuation of Deferred@1447010749 after retry RPC after error@1596824163) -> (continuation of Deferred@981600834 after retry RPC after error@668640861) -> (continuation of Deferred@205769042 after retry RPC after error@1066316859) -> (continuation of Deferred@368343442 after retry RPC after error@1735314751) -> (continuation of Deferred@1970635225 after retry RPC after error@1058069536) -> (continuation of Deferred@152967865 after retry RPC after error@411580816) -> (continuation of Deferred@1101662225 after retry RPC after error@2019700184) -> (continuation of Deferred@30423999 after retry RPC after error@1489880748) -> (continuation of Deferred@1228318500 after retry RPC after error@542330679) -> (continuation of Deferred@551454017 after retry RPC after error@370200400) -> (continuation of Deferred@1793283377 after retry RPC after error@1349748141) -> (continuation of Deferred@80995951 after retry RPC after error@346674154) -> (continuation of Deferred@1687377555 after retry RPC after error@679606593), errback=(continuation of Deferred@663248745 after retry RPC after error@1494206213) -> (continuation of Deferred@1714759237 after retry RPC after error@391938415) -> (continuation of Deferred@1917511674 after retry RPC after error@1440362841) -> (continuation of Deferred@437396137 after retry RPC after error@1773571965) -> (continuation of Deferred@1290195078 after retry RPC after error@1219649722) -> (continuation of Deferred@1128141935 after retry RPC after error@514766354) -> (continuation of Deferred@1681274785 after retry RPC after error@1071262765) -> (continuation of Deferred@217369301 after retry RPC after error@1736569727) -> (continuation of Deferred@845105937 after retry RPC after error@1552593459) -> (continuation of Deferred@1988262165 after retry RPC after error@519075505) -> (continuation of Deferred@1447010749 after retry RPC after error@1596824163) -> (continuation of Deferred@981600834 after retry RPC after error@668640861) -> (continuation of Deferred@205769042 after retry RPC after error@1066316859) -> (continuation of Deferred@368343442 after retry RPC after error@1735314751) -> (continuation of Deferred@1970635225 after retry RPC after error@1058069536) -> (continuation of Deferred@152967865 after retry RPC after error@411580816) -> (continuation of Deferred@1101662225 after retry RPC after error@2019700184) -> (continuation of Deferred@30423999 after retry RPC after error@1489880748) -> (continuation of Deferred@1228318500 after retry RPC after error@542330679) -> (continuation of Deferred@551454017 after retry RPC after error@370200400) -> (continuation of Deferred@1793283377 after retry RPC after error@1349748141) -> (continuation of Deferred@80995951 after retry RPC after error@346674154) -> (continuation of Deferred@1687377555 after retry RPC after error@679606593)))
20:26:49.417 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.93.125:36801(127.28.93.125:36801)] disconnecting while in state TERMINATED
20:26:49.418 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.93.124:39973(127.28.93.124:39973)] disconnecting while in state READY
20:26:49.419 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.93.126:37603(127.28.93.126:37603)] disconnecting while in state TERMINATED
20:26:49.424 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:26:49.428 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:26:49.428 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.93.124:39973(127.28.93.124:39973)] handling channelInactive
20:26:49.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.93.124:39973(127.28.93.124:39973)] cleaning up while in state READY due to: connection closed
20:26:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:49.431351 29045 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 29571
20:26:50.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:26:50Z chronyd exiting
20:26:51.567 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:26:51.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:51.568248 28002 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 28395
20:26:51.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:51.619467 28002 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 28638
20:26:51.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:51.661695 28002 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 28850
20:26:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:51.704314 28002 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 28020
20:26:51.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:51.738969 28002 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 28115
20:26:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260806 20:26:51.776314 28002 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskIlcT6h/build/tsan/bin/kudu with pid 28218
20:26:52.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-06T20:26:52Z chronyd exiting
20:26:53.883 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29930), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [194ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [197ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [583ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [605ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [606ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [608ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [608ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [619ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [624ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [626ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [639ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [640ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [642ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [643ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [660ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [660ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [662ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [679ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [679ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [683ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [742ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [744ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [802ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [803ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [835ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [839ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [841ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [843ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [844ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [1315ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [1323ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [2355ms] refreshing cache from master, [2355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [2357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [2361ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [2361ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [3795ms] refreshing cache from master, [3795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [3796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [3797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [3800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [3800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [3801ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [3801ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [5875ms] refreshing cache from master, [5876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [5876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [5877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [5880ms] too many traces: truncated at 100 traces, deferred=Deferred@1665289055(state=PENDING, result=null, callback=(continuation of Deferred@663248745 after retry RPC after error@1494206213) -> (continuation of Deferred@1714759237 after retry RPC after error@391938415) -> (continuation of Deferred@1917511674 after retry RPC after error@1440362841) -> (continuation of Deferred@437396137 after retry RPC after error@1773571965) -> (continuation of Deferred@1290195078 after retry RPC after error@1219649722) -> (continuation of Deferred@1128141935 after retry RPC after error@514766354) -> (continuation of Deferred@1681274785 after retry RPC after error@1071262765) -> (continuation of Deferred@217369301 after retry RPC after error@1736569727) -> (continuation of Deferred@845105937 after retry RPC after error@1552593459) -> (continuation of Deferred@1988262165 after retry RPC after error@519075505) -> (continuation of Deferred@1447010749 after retry RPC after error@1596824163) -> (continuation of Deferred@981600834 after retry RPC after error@668640861) -> (continuation of Deferred@205769042 after retry RPC after error@1066316859) -> (continuation of Deferred@368343442 after retry RPC after error@1735314751) -> (continuation of Deferred@1970635225 after retry RPC after error@1058069536) -> (continuation of Deferred@152967865 after retry RPC after error@411580816) -> (continuation of Deferred@1101662225 after retry RPC after error@2019700184) -> (continuation of Deferred@30423999 after retry RPC after error@1489880748) -> (continuation of Deferred@1228318500 after retry RPC after error@542330679) -> (continuation of Deferred@551454017 after retry RPC after error@370200400) -> (continuation of Deferred@1793283377 after retry RPC after error@1349748141) -> (continuation of Deferred@80995951 after retry RPC after error@346674154) -> (continuation of Deferred@1687377555 after retry RPC after error@679606593), errback=(continuation of Deferred@663248745 after retry RPC after error@1494206213) -> (continuation of Deferred@1714759237 after retry RPC after error@391938415) -> (continuation of Deferred@1917511674 after retry RPC after error@1440362841) -> (continuation of Deferred@437396137 after retry RPC after error@1773571965) -> (continuation of Deferred@1290195078 after retry RPC after error@1219649722) -> (continuation of Deferred@1128141935 after retry RPC after error@514766354) -> (continuation of Deferred@1681274785 after retry RPC after error@1071262765) -> (continuation of Deferred@217369301 after retry RPC after error@1736569727) -> (continuation of Deferred@845105937 after retry RPC after error@1552593459) -> (continuation of Deferred@1988262165 after retry RPC after error@519075505) -> (continuation of Deferred@1447010749 after retry RPC after error@1596824163) -> (continuation of Deferred@981600834 after retry RPC after error@668640861) -> (continuation of Deferred@205769042 after retry RPC after error@1066316859) -> (continuation of Deferred@368343442 after retry RPC after error@1735314751) -> (continuation of Deferred@1970635225 after retry RPC after error@1058069536) -> (continuation of Deferred@152967865 after retry RPC after error@411580816) -> (continuation of Deferred@1101662225 after retry RPC after error@2019700184) -> (continuation of Deferred@30423999 after retry RPC after error@1489880748) -> (continuation of Deferred@1228318500 after retry RPC after error@542330679) -> (continuation of Deferred@551454017 after retry RPC after error@370200400) -> (continuation of Deferred@1793283377 after retry RPC after error@1349748141) -> (continuation of Deferred@80995951 after retry RPC after error@346674154) -> (continuation of Deferred@1687377555 after retry RPC after error@679606593)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.28.93.125:36801
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=24, TimeoutTracker(timeout=30000, elapsed=29930), Traces: [0ms] refreshing cache from master, [58ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [194ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [197ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [583ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [587ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [595ms] refreshing cache from master, [596ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [599ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [605ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [606ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [608ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [608ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [615ms] refreshing cache from master, [615ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [616ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [619ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [624ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [626ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [635ms] refreshing cache from master, [635ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [639ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [640ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [642ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [643ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [655ms] refreshing cache from master, [655ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [656ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [660ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [660ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [662ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [663ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [675ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [679ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [679ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [683ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [735ms] refreshing cache from master, [735ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [736ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [739ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [742ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [744ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [795ms] refreshing cache from master, [795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [802ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [803ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [835ms] refreshing cache from master, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [836ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [837ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [839ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [841ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [843ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [844ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [1315ms] refreshing cache from master, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [1316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [1320ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [1323ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [2355ms] refreshing cache from master, [2355ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [2356ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [2357ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [2359ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [2361ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [2361ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [3795ms] refreshing cache from master, [3795ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [3796ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [3797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [3800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.125:36801: Network error: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [3800ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.126:37603: Network error: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603, [3801ms] Sub RPC ConnectToMaster: received response from server master-127.28.93.124:39973: OK, [3801ms] delaying RPC due to: Service unavailable: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801, [5875ms] refreshing cache from master, [5876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.126:37603, [5876ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.124:39973, [5877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.93.125:36801, [5880ms] too many traces: truncated at 100 traces, deferred=Deferred@1665289055(state=PENDING, result=null, callback=(continuation of Deferred@663248745 after retry RPC after error@1494206213) -> (continuation of Deferred@1714759237 after retry RPC after error@391938415) -> (continuation of Deferred@1917511674 after retry RPC after error@1440362841) -> (continuation of Deferred@437396137 after retry RPC after error@1773571965) -> (continuation of Deferred@1290195078 after retry RPC after error@1219649722) -> (continuation of Deferred@1128141935 after retry RPC after error@514766354) -> (continuation of Deferred@1681274785 after retry RPC after error@1071262765) -> (continuation of Deferred@217369301 after retry RPC after error@1736569727) -> (continuation of Deferred@845105937 after retry RPC after error@1552593459) -> (continuation of Deferred@1988262165 after retry RPC after error@519075505) -> (continuation of Deferred@1447010749 after retry RPC after error@1596824163) -> (continuation of Deferred@981600834 after retry RPC after error@668640861) -> (continuation of Deferred@205769042 after retry RPC after error@1066316859) -> (continuation of Deferred@368343442 after retry RPC after error@1735314751) -> (continuation of Deferred@1970635225 after retry RPC after error@1058069536) -> (continuation of Deferred@152967865 after retry RPC after error@411580816) -> (continuation of Deferred@1101662225 after retry RPC after error@2019700184) -> (continuation of Deferred@30423999 after retry RPC after error@1489880748) -> (continuation of Deferred@1228318500 after retry RPC after error@542330679) -> (continuation of Deferred@551454017 after retry RPC after error@370200400) -> (continuation of Deferred@1793283377 after retry RPC after error@1349748141) -> (continuation of Deferred@80995951 after retry RPC after error@346674154) -> (continuation of Deferred@1687377555 after retry RPC after error@679606593), errback=(continuation of Deferred@663248745 after retry RPC after error@1494206213) -> (continuation of Deferred@1714759237 after retry RPC after error@391938415) -> (continuation of Deferred@1917511674 after retry RPC after error@1440362841) -> (continuation of Deferred@437396137 after retry RPC after error@1773571965) -> (continuation of Deferred@1290195078 after retry RPC after error@1219649722) -> (continuation of Deferred@1128141935 after retry RPC after error@514766354) -> (continuation of Deferred@1681274785 after retry RPC after error@1071262765) -> (continuation of Deferred@217369301 after retry RPC after error@1736569727) -> (continuation of Deferred@845105937 after retry RPC after error@1552593459) -> (continuation of Deferred@1988262165 after retry RPC after error@519075505) -> (continuation of Deferred@1447010749 after retry RPC after error@1596824163) -> (continuation of Deferred@981600834 after retry RPC after error@668640861) -> (continuation of Deferred@205769042 after retry RPC after error@1066316859) -> (continuation of Deferred@368343442 after retry RPC after error@1735314751) -> (continuation of Deferred@1970635225 after retry RPC after error@1058069536) -> (continuation of Deferred@152967865 after retry RPC after error@411580816) -> (continuation of Deferred@1101662225 after retry RPC after error@2019700184) -> (continuation of Deferred@30423999 after retry RPC after error@1489880748) -> (continuation of Deferred@1228318500 after retry RPC after error@542330679) -> (continuation of Deferred@551454017 after retry RPC after error@370200400) -> (continuation of Deferred@1793283377 after retry RPC after error@1349748141) -> (continuation of Deferred@80995951 after retry RPC after error@346674154) -> (continuation of Deferred@1687377555 after retry RPC after error@679606593)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.28.93.126:37603,127.28.93.124:39973,127.28.93.125:36801) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.126:37603(127.28.93.126:37603): Connection refused: /127.28.93.126:37603,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.28.93.125:36801(127.28.93.125:36801): Connection refused: /127.28.93.125:36801
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.access$200(DefaultPromise.java:35)
at io.netty.util.concurrent.DefaultPromise$1.run(DefaultPromise.java:503)
at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:569)
... 5 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.28.93.125:36801
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)