07:45:52.406 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
07:45:52.418 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHOPlnL/test-tmp
07:45:52.421 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:45:52.424 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHOPlnL/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:45:52.669 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
07:45:52.877 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:45:52.878 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:45:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:52.883037 25440 mini_oidc.cc:152] Starting JWKS server
07:45:52.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:52.888458 25440 webserver.cc:460] Webserver started at http://127.0.0.1:45027/ using document root <none> and password file <none>
07:45:52.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:52.889144 25440 mini_oidc.cc:189] Starting OIDC Discovery server
07:45:52.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:52.891321 25440 webserver.cc:460] Webserver started at http://127.0.0.1:42493/ using document root <none> and password file <none>
07:45:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T07:45:52Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:45:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T07:45:52Z Disabled control of system clock
07:45:52.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:52.957520 25440 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:45:52.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:45:52.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/wal
07:45:52.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/data
07:45:52.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:45:52.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:45:52.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:45:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:45:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:45:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:45:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/logs
07:45:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/data/info.pb
07:45:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:45:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:45:52.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:45:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:45:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:45:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:45:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:45:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:45:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:45:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:45:52.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:45:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:45:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.62:36897
07:45:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.62
07:45:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:45:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:45:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:45:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:45:52.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:45:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:45:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:45:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:45:52.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json with env {}
07:45:53.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.291885 25452 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:45:53.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.292443 25452 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:45:53.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.293087 25452 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:45:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.331140 25452 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:45:53.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.331418 25452 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:45:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.331609 25452 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:45:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.331815 25452 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:45:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.332098 25452 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:45:53.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.332319 25452 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45027/jwks.json
07:45:53.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:53.367213 25452 master_runner.cc:386] Master server non-default flags:
07:45:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:45:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:45:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:45:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:45:53.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/data
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/wal
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.62:36897
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:45:53.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/data/info.pb
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.62
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:45:53.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/logs
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:53.368800 25452 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:45:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:53.371006 25452 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:45:53.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.390043 25459 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.390177 25461 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:54.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:53.390210 25458 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:54.658815 25460 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1269 milliseconds
07:45:54.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.658942 25452 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:45:54.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.662580 25452 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:45:54.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.665732 25452 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:45:54.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.667186 25452 hybrid_clock.cc:648] HybridClock initialized: now 1790408754667153 us; error 53 us; skew 500 ppm
07:45:54.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.668164 25452 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:45:54.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.674949 25452 webserver.cc:460] Webserver started at http://127.24.216.62:44023/ using document root <none> and password file <none>
07:45:54.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.675892 25452 fs_manager.cc:359] Metadata directory not provided
07:45:54.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.676081 25452 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:45:54.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.676498 25452 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:45:54.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.681492 25452 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/data/instance:
07:45:54.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88c603f733444a4e8416378cb19dc650"
07:45:54.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:54 on dist-test-slave-f0nh"
07:45:54.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.682788 25452 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/wal/instance:
07:45:54.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88c603f733444a4e8416378cb19dc650"
07:45:54.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:54 on dist-test-slave-f0nh"
07:45:54.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.690009 25452 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
07:45:54.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.695652 25468 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:45:54.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.696992 25452 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:45:54.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.697336 25452 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/wal
07:45:54.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "88c603f733444a4e8416378cb19dc650"
07:45:54.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:54 on dist-test-slave-f0nh"
07:45:54.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.697810 25452 fs_report.cc:383] FS layout report
07:45:54.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:45:54.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/wal
07:45:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/wal
07:45:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/data/data
07:45:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:45:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:45:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:45:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:45:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:45:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:45:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:45:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:45:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:45:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:45:54.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:45:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.746716 25452 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:45:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.748467 25452 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:45:54.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.749030 25452 kserver.cc:163] Server-wide thread pool size limit: 3276
07:45:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.827641 25452 rpc_server.cc:307] RPC server started. Bound to: 127.24.216.62:36897
07:45:54.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.827735 25519 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.216.62:36897 every 8 connection(s)
07:45:54.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.830601 25452 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/data/info.pb
07:45:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.836220 25520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:45:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.839973 25440 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 25452
07:45:54.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.840364 25440 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-0/wal/instance
07:45:54.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.844525 25440 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/wal
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/data
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:45:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/logs
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/data/info.pb
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:45:54.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:45:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:45:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:45:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.61:41609
07:45:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.61
07:45:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:45:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:45:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:45:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:45:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:45:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:45:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:45:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:45:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json with env {}
07:45:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.849574 25520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } has no permanent_uuid. Determining permanent_uuid...
07:45:54.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.878979 25520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } has no permanent_uuid. Determining permanent_uuid...
07:45:54.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:54.881167 25469 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.216.61:41609: connect: Connection refused (error 111)
07:45:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:54.885214 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.61:41609: Network error: Client connection negotiation failed: client connection to 127.24.216.61:41609: connect: Connection refused (error 111)
07:45:54.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:54.934670 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } attempt: 1
07:45:54.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:54.939482 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.61:41609: Network error: Client connection negotiation failed: client connection to 127.24.216.61:41609: connect: Connection refused (error 111)
07:45:55.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:55.007848 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } attempt: 2
07:45:55.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.014897 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.61:41609: Network error: Client connection negotiation failed: client connection to 127.24.216.61:41609: connect: Connection refused (error 111)
07:45:55.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:55.106403 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } attempt: 3
07:45:55.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.110891 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.61:41609: Network error: Client connection negotiation failed: client connection to 127.24.216.61:41609: connect: Connection refused (error 111)
07:45:55.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.186089 25522 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:45:55.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.186622 25522 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:45:55.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.187229 25522 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:45:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.218021 25522 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:45:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.218308 25522 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:45:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.218524 25522 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:45:55.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.218703 25522 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:45:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.218995 25522 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:45:55.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.219261 25522 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45027/jwks.json
07:45:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:55.254132 25522 master_runner.cc:386] Master server non-default flags:
07:45:55.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/data
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/wal
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:45:55.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.61:41609
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/data/info.pb
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.61
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:45:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:45:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:45:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:45:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:45:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:45:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/logs
07:45:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:45:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:45:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:45:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:45:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:45:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:45:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:45:55.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:45:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:45:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:45:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:55.254276 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } attempt: 4
07:45:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:55.255749 25522 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:45:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:55.258196 25522 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:45:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.260067 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.61:41609: Network error: Client connection negotiation failed: client connection to 127.24.216.61:41609: connect: Connection refused (error 111)
07:45:55.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.280665 25534 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:55.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:55.559484 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } attempt: 5
07:45:55.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.562960 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.61:41609: Network error: Client connection negotiation failed: client connection to 127.24.216.61:41609: connect: Connection refused (error 111)
07:45:56.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.110574 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } attempt: 6
07:45:56.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:56.117832 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.61:41609: Network error: Client connection negotiation failed: client connection to 127.24.216.61:41609: connect: Connection refused (error 111)
07:45:56.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.280790 25532 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:56.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:55.280812 25531 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:56.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:55.282351 25522 server_base.cc:1034] running on GCE node
07:45:56.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.520987 25522 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:45:56.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.524281 25522 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:45:56.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.525718 25522 hybrid_clock.cc:648] HybridClock initialized: now 1790408756525659 us; error 76 us; skew 500 ppm
07:45:56.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.526535 25522 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:45:56.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.540506 25522 webserver.cc:460] Webserver started at http://127.24.216.61:44549/ using document root <none> and password file <none>
07:45:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.541478 25522 fs_manager.cc:359] Metadata directory not provided
07:45:56.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.541661 25522 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:45:56.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.542034 25522 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:45:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.547384 25522 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/data/instance:
07:45:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d06a17f134a45998a62405b1990fa02"
07:45:56.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:56 on dist-test-slave-f0nh"
07:45:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.548983 25522 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/wal/instance:
07:45:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d06a17f134a45998a62405b1990fa02"
07:45:56.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:56 on dist-test-slave-f0nh"
07:45:56.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.556998 25522 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
07:45:56.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.562356 25543 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:45:56.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.563503 25522 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
07:45:56.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.563781 25522 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/wal
07:45:56.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8d06a17f134a45998a62405b1990fa02"
07:45:56.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:56 on dist-test-slave-f0nh"
07:45:56.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.564075 25522 fs_report.cc:383] FS layout report
07:45:56.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:45:56.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/wal
07:45:56.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/wal
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/data/data
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:45:56.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:45:56.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.601692 25522 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:45:56.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.603734 25522 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:45:56.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.604285 25522 kserver.cc:163] Server-wide thread pool size limit: 3276
07:45:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.720427 25522 rpc_server.cc:307] RPC server started. Bound to: 127.24.216.61:41609
07:45:56.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.720501 25594 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.216.61:41609 every 8 connection(s)
07:45:56.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.723052 25522 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/data/info.pb
07:45:56.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.729072 25595 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:45:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.729854 25440 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 25522
07:45:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.730274 25440 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-1/wal/instance
07:45:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.732438 25440 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:45:56.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/wal
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/data
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/logs
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/data/info.pb
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:45:56.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.60:46401
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.60
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:45:56.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:45:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:45:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:45:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:45:56.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json with env {}
07:45:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.740986 25595 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } has no permanent_uuid. Determining permanent_uuid...
07:45:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.770812 25595 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } has no permanent_uuid. Determining permanent_uuid...
07:45:56.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.786584 25595 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } has no permanent_uuid. Determining permanent_uuid...
07:45:56.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:56.789076 25547 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:56.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:56.794112 25595 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:56.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.843557 25595 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 1
07:45:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:56.847432 25595 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:56.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:56.915788 25595 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 2
07:45:56.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:56.922418 25595 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.013891 25595 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 3
07:45:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.017603 25595 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.056772 25597 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:45:57.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.057330 25597 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:45:57.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.057782 25597 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:45:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.089346 25597 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:45:57.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.089614 25597 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:45:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.089805 25597 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:45:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.090112 25597 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:45:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.090442 25597 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:45:57.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.090695 25597 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:45027/jwks.json
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.124980 25597 master_runner.cc:386] Master server non-default flags:
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/data
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/wal
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:45:57.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.60:46401
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:45027/jwks.json
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/data/info.pb
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.60
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:45:57.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/logs
07:45:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:45:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:45:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:45:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:45:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:45:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:45:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:45:57.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:45:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:45:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:45:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.126245 25597 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:45:57.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.128168 25597 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:45:57.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.146199 25610 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:57.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.161000 25595 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 4
07:45:57.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.166730 25595 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.178494 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } attempt: 7
07:45:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.201633 25520 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } has no permanent_uuid. Determining permanent_uuid...
07:45:57.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.210587 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.269240 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 1
07:45:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.275370 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.356961 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 2
07:45:57.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.362654 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.448597 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 3
07:45:57.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.454398 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.466141 25595 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 5
07:45:57.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.472193 25595 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.594995 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 4
07:45:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.600765 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:57.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:57.884297 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 5
07:45:57.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.891892 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:58.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.019762 25595 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 6
07:45:58.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:58.032532 25595 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:58.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.146283 25607 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:58.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:57.146780 25608 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:45:58.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:58.396498 25609 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1248 milliseconds
07:45:58.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.396621 25597 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:45:58.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.397984 25597 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:45:58.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.400511 25597 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:45:58.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.401892 25597 hybrid_clock.cc:648] HybridClock initialized: now 1790408758401862 us; error 48 us; skew 500 ppm
07:45:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.402770 25597 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:45:58.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.411377 25597 webserver.cc:460] Webserver started at http://127.24.216.60:37961/ using document root <none> and password file <none>
07:45:58.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.412451 25597 fs_manager.cc:359] Metadata directory not provided
07:45:58.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.412946 25597 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:45:58.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.413609 25597 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:45:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.418776 25597 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/data/instance:
07:45:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af2e752f2a2143a78d9954ae6f64ae7e"
07:45:58.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:58 on dist-test-slave-f0nh"
07:45:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.419900 25597 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/wal/instance:
07:45:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af2e752f2a2143a78d9954ae6f64ae7e"
07:45:58.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:58 on dist-test-slave-f0nh"
07:45:58.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.428686 25597 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.005s
07:45:58.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.435482 25621 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:45:58.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.436487 25597 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
07:45:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.436865 25597 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/wal
07:45:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "af2e752f2a2143a78d9954ae6f64ae7e"
07:45:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:45:58 on dist-test-slave-f0nh"
07:45:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.437233 25597 fs_report.cc:383] FS layout report
07:45:58.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/wal
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/wal
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/data/data
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:45:58.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:45:58.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.444537 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 6
07:45:58.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:58.451005 25520 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.24.216.60:46401: Network error: Client connection negotiation failed: client connection to 127.24.216.60:46401: connect: Connection refused (error 111)
07:45:58.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.508901 25597 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:45:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.510553 25597 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:45:58.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.511098 25597 kserver.cc:163] Server-wide thread pool size limit: 3276
07:45:58.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.597690 25597 rpc_server.cc:307] RPC server started. Bound to: 127.24.216.60:46401
07:45:58.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.597793 25674 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.216.60:46401 every 8 connection(s)
07:45:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.600476 25597 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/data/info.pb
07:45:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.604760 25440 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 25597
07:45:58.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.605218 25440 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/master-2/wal/instance
07:45:58.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.606043 25675 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:45:58.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.623185 25675 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } has no permanent_uuid. Determining permanent_uuid...
07:45:58.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.649714 25675 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } has no permanent_uuid. Determining permanent_uuid...
07:45:58.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.661840 25675 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } has no permanent_uuid. Determining permanent_uuid...
07:45:58.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.682791 25675 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e: Bootstrap starting.
07:45:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.689347 25675 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e: Neither blocks nor log segments found. Creating new log.
07:45:58.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.691594 25675 log.cc:824] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e: Log is configured to *not* fsync() on all Append() calls
07:45:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.695791 25675 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e: No bootstrap required, opened a new log
07:45:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.717407 25675 raft_consensus.cc:348] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.718153 25675 raft_consensus.cc:374] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:45:58.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.718343 25675 raft_consensus.cc:729] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af2e752f2a2143a78d9954ae6f64ae7e, State: Initialized, Role: FOLLOWER
07:45:58.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.719043 25675 consensus_queue.cc:260] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:58.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.724921 25684 sys_catalog.cc:455] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:58.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.725793 25684 sys_catalog.cc:458] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: This master's current role is: FOLLOWER
07:45:58.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.726591 25675 sys_catalog.cc:564] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: configured and running, proceeding with master startup.
07:45:58.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.733551 25684 raft_consensus.cc:482] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:45:58.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.734112 25684 raft_consensus.cc:504] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:58.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.741070 25684 leader_election.cc:288] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 88c603f733444a4e8416378cb19dc650 (127.24.216.62:36897), 8d06a17f134a45998a62405b1990fa02 (127.24.216.61:41609)
07:45:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:58.763756 25695 catalog_manager.cc:1558] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:45:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:58.764052 25695 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:45:58.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:58.766137 25629 tablet.cc:2369] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:45:58.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.761754 25597 master_runner.cc:186] Error getting master registration for 127.24.216.62:36897: OK, instance_id { permanent_uuid: "88c603f733444a4e8416378cb19dc650" instance_seqno: 1790408754806197 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:45:58.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:58.796918 25597 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:45:59.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.093147 25595 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 7
07:45:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.113055 25595 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: Bootstrap starting.
07:45:59.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.120175 25595 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: Neither blocks nor log segments found. Creating new log.
07:45:59.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.122428 25595 log.cc:824] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: Log is configured to *not* fsync() on all Append() calls
07:45:59.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.129092 25595 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: No bootstrap required, opened a new log
07:45:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.155124 25595 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.156002 25595 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:45:59.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.156312 25595 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8d06a17f134a45998a62405b1990fa02, State: Initialized, Role: FOLLOWER
07:45:59.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.157591 25595 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:59.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.163797 25699 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:59.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.164794 25699 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [sys.catalog]: This master's current role is: FOLLOWER
07:45:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.165954 25595 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [sys.catalog]: configured and running, proceeding with master startup.
07:45:59.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.166172 25570 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8d06a17f134a45998a62405b1990fa02" is_pre_election: true
07:45:59.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.167173 25570 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af2e752f2a2143a78d9954ae6f64ae7e in term 0.
07:45:59.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.168467 25624 leader_election.cc:302] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8d06a17f134a45998a62405b1990fa02, af2e752f2a2143a78d9954ae6f64ae7e; no voters:
07:45:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.169828 25684 raft_consensus.cc:2781] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 0 FOLLOWER]: Leader pre-election won for term 1
07:45:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.170166 25684 raft_consensus.cc:482] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
07:45:59.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.170563 25684 raft_consensus.cc:3037] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 0 FOLLOWER]: Advancing to term 1
07:45:59.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.178974 25684 raft_consensus.cc:504] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:59.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.182119 25684 leader_election.cc:288] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [CANDIDATE]: Term 1 election: Requested vote from peers 88c603f733444a4e8416378cb19dc650 (127.24.216.62:36897), 8d06a17f134a45998a62405b1990fa02 (127.24.216.61:41609)
07:45:59.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.183462 25570 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8d06a17f134a45998a62405b1990fa02"
07:45:59.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.184293 25570 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [term 0 FOLLOWER]: Advancing to term 1
07:45:59.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.192343 25570 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate af2e752f2a2143a78d9954ae6f64ae7e in term 1.
07:45:59.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.194469 25624 leader_election.cc:302] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8d06a17f134a45998a62405b1990fa02, af2e752f2a2143a78d9954ae6f64ae7e; no voters:
07:45:59.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.195498 25684 raft_consensus.cc:2781] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 1 FOLLOWER]: Leader election won for term 1
07:45:59.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.205617 25684 raft_consensus.cc:686] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [term 1 LEADER]: Becoming Leader. State: Replica: af2e752f2a2143a78d9954ae6f64ae7e, State: Running, Role: LEADER
07:45:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.207242 25684 consensus_queue.cc:237] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:59.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.231233 25705 sys_catalog.cc:455] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: SysCatalogTable state changed. Reason: New leader af2e752f2a2143a78d9954ae6f64ae7e. Latest consensus state: current_term: 1 leader_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.232560 25705 sys_catalog.cc:458] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: This master's current role is: LEADER
07:45:59.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.245211 25711 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
07:45:59.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:59.276818 25713 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:45:59.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.277050 25711 catalog_manager.cc:1484] Initializing Kudu cluster ID...
07:45:59.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:59.277632 25713 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:45:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.304965 25570 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [term 1 FOLLOWER]: Refusing update from remote peer af2e752f2a2143a78d9954ae6f64ae7e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:45:59.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.307878 25705 consensus_queue.cc:1035] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [LEADER]: Connected to new peer: Peer: permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
07:45:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.250541 25522 master_runner.cc:186] Error getting master registration for 127.24.216.62:36897: OK, instance_id { permanent_uuid: "88c603f733444a4e8416378cb19dc650" instance_seqno: 1790408754806197 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:45:59.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.361373 25522 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:45:59.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:45:59.365676 25549 tablet.cc:2369] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:45:59.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.377947 25715 mvcc.cc:205] Tried to move back new op lower bound from 7333514278088413184 to 7333514277729529856. Current Snapshot: MvccSnapshot[applied={T|T < 7333514278088413184}]
07:45:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.383193 25699 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [sys.catalog]: SysCatalogTable state changed. Reason: New leader af2e752f2a2143a78d9954ae6f64ae7e. Latest consensus state: current_term: 1 leader_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:59.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.384099 25699 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [sys.catalog]: This master's current role is: FOLLOWER
07:45:59.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.389660 25684 sys_catalog.cc:455] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:59.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.390399 25684 sys_catalog.cc:458] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: This master's current role is: LEADER
07:45:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.393456 25711 catalog_manager.cc:1347] Generated new cluster ID: 43eccca5967145329d6f71afc2f7352d
07:45:59.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.393728 25711 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
07:45:59.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.420022 25699 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:59.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.420801 25699 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02 [sys.catalog]: This master's current role is: FOLLOWER
07:45:59.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.430553 25711 catalog_manager.cc:1370] Generated new certificate authority record
07:45:59.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.433234 25711 catalog_manager.cc:1504] Loading token signing keys...
07:45:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.462209 25711 catalog_manager.cc:5868] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e: Generated new TSK 0
07:45:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.466936 25711 catalog_manager.cc:1514] Initializing in-progress tserver states...
07:45:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.484337 25520 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } attempt: 7
07:45:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.503890 25440 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:45:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:45:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/wal
07:45:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/data
07:45:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:45:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/logs
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/data/info.pb
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.1:0
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.216.1
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.1
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:45:59.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:45:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:45:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:45:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.510315 25520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650: Bootstrap starting.
07:45:59.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.518267 25520 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650: Neither blocks nor log segments found. Creating new log.
07:45:59.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.520911 25520 log.cc:824] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650: Log is configured to *not* fsync() on all Append() calls
07:45:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.532267 25520 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650: No bootstrap required, opened a new log
07:45:59.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.570914 25520 raft_consensus.cc:348] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:59.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.572247 25520 raft_consensus.cc:374] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:45:59.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.572851 25520 raft_consensus.cc:729] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 88c603f733444a4e8416378cb19dc650, State: Initialized, Role: FOLLOWER
07:45:59.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.574186 25520 consensus_queue.cc:260] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } }
07:45:59.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.586242 25520 sys_catalog.cc:564] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [sys.catalog]: configured and running, proceeding with master startup.
07:45:59.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.587616 25493 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [term 0 FOLLOWER]: Advancing to term 1
07:45:59.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.593012 25494 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88c603f733444a4e8416378cb19dc650"
07:45:59.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.586427 25495 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "88c603f733444a4e8416378cb19dc650" is_pre_election: true
07:45:59.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.602782 25493 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [term 1 FOLLOWER]: Refusing update from remote peer af2e752f2a2143a78d9954ae6f64ae7e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
07:45:59.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.607059 25684 consensus_queue.cc:1035] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [LEADER]: Connected to new peer: Peer: permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
07:45:59.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.610823 25724 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:59.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.611968 25724 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [sys.catalog]: This master's current role is: FOLLOWER
07:45:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.677409 25734 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.678315 25734 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [sys.catalog]: This master's current role is: FOLLOWER
07:45:59.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.679184 25724 sys_catalog.cc:455] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [sys.catalog]: SysCatalogTable state changed. Reason: New leader af2e752f2a2143a78d9954ae6f64ae7e. Latest consensus state: current_term: 1 leader_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:45:59.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.680071 25724 sys_catalog.cc:458] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650 [sys.catalog]: This master's current role is: FOLLOWER
07:45:59.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:45:59.689553 25740 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
07:46:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:00.040786 25722 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:00.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:00.041393 25722 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:00.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:00.041985 25722 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:00.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:00.073381 25722 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:00.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:00.074366 25722 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.216.1
07:46:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.110931 25722 tablet_server_runner.cc:78] Tablet server non-default flags:
07:46:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:46:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/data
07:46:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/wal
07:46:00.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.1:0
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/data/info.pb
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.1
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:00.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.216.1
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/logs
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:00.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.112480 25722 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:00.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.114467 25722 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:00.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:00.138243 25746 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:00.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.227492 25684 sys_catalog.cc:455] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "88c603f733444a4e8416378cb19dc650" member_type: VOTER last_known_addr { host: "127.24.216.62" port: 36897 } } peers { permanent_uuid: "8d06a17f134a45998a62405b1990fa02" member_type: VOTER last_known_addr { host: "127.24.216.61" port: 41609 } } peers { permanent_uuid: "af2e752f2a2143a78d9954ae6f64ae7e" member_type: VOTER last_known_addr { host: "127.24.216.60" port: 46401 } } }
07:46:00.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.228853 25684 sys_catalog.cc:458] T 00000000000000000000000000000000 P af2e752f2a2143a78d9954ae6f64ae7e [sys.catalog]: This master's current role is: LEADER
07:46:00.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.284281 25713 catalog_manager.cc:1259] Loaded cluster ID: 43eccca5967145329d6f71afc2f7352d
07:46:00.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.284920 25713 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: loading cluster ID for follower catalog manager: success
07:46:00.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.300678 25713 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: acquiring CA information for follower catalog manager: success
07:46:00.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.314146 25713 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8d06a17f134a45998a62405b1990fa02: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:46:00.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.709326 25740 catalog_manager.cc:1259] Loaded cluster ID: 43eccca5967145329d6f71afc2f7352d
07:46:00.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.715731 25740 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650: loading cluster ID for follower catalog manager: success
07:46:00.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.724509 25740 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650: acquiring CA information for follower catalog manager: success
07:46:00.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:00.739310 25740 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 88c603f733444a4e8416378cb19dc650: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
07:46:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:01.531827 25745 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 25722
07:46:02.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:02.737406 25722 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.600s user 0.603s sys 1.088s
07:46:02.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:00.138916 25747 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:02.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:02.747243 25749 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:02.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:02.747951 25722 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.611s user 0.603s sys 1.088s
07:46:02.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.756232 25722 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:46:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:02.756959 25748 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2614 milliseconds
07:46:02.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.759004 25722 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:02.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.768901 25722 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:02.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.776805 25722 hybrid_clock.cc:648] HybridClock initialized: now 1790408762776750 us; error 70 us; skew 500 ppm
07:46:02.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.777909 25722 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:02.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.810907 25722 webserver.cc:460] Webserver started at http://127.24.216.1:37185/ using document root <none> and password file <none>
07:46:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.812152 25722 fs_manager.cc:359] Metadata directory not provided
07:46:02.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.812423 25722 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.812988 25722 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.821412 25722 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/data/instance:
07:46:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e41c73815c14d81b0d1cb1c71d8177f"
07:46:02.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:02 on dist-test-slave-f0nh"
07:46:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.823086 25722 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/wal/instance:
07:46:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e41c73815c14d81b0d1cb1c71d8177f"
07:46:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:02 on dist-test-slave-f0nh"
07:46:02.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.838354 25722 fs_manager.cc:691] Time spent creating directory manager: real 0.014s user 0.002s sys 0.014s
07:46:02.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.851310 25760 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:02.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.852517 25722 fs_manager.cc:725] Time spent opening block manager: real 0.010s user 0.002s sys 0.008s
07:46:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.852922 25722 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/wal
07:46:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e41c73815c14d81b0d1cb1c71d8177f"
07:46:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:02 on dist-test-slave-f0nh"
07:46:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.853333 25722 fs_report.cc:383] FS layout report
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/wal
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/wal
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/data/data
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:02.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:02.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.931089 25722 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:02.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.932960 25722 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:02.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.933511 25722 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:02.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.937122 25722 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:46:02.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.943082 25722 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:46:02.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.943358 25722 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:46:02.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.943655 25722 ts_tablet_manager.cc:610] Registered 0 tablets
07:46:02.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:02.943867 25722 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:46:03.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.147841 25722 rpc_server.cc:307] RPC server started. Bound to: 127.24.216.1:33621
07:46:03.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.147915 25872 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.216.1:33621 every 8 connection(s)
07:46:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.151367 25722 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/data/info.pb
07:46:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.160842 25440 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 25722
07:46:03.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.161573 25440 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-0/wal/instance
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.176532 25440 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/wal
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/data
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/logs
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/data/info.pb
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.2:0
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.216.2
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.2
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:03.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:46:03.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.218325 25873 heartbeater.cc:347] Connected to a master server at 127.24.216.61:41609
07:46:03.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.218737 25873 heartbeater.cc:464] Registering TS with master...
07:46:03.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.219901 25873 heartbeater.cc:511] Master 127.24.216.61:41609 requested a full tablet report, sending...
07:46:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.221536 25874 heartbeater.cc:347] Connected to a master server at 127.24.216.60:46401
07:46:03.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.221992 25874 heartbeater.cc:464] Registering TS with master...
07:46:03.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.222893 25874 heartbeater.cc:511] Master 127.24.216.60:46401 requested a full tablet report, sending...
07:46:03.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.223574 25560 ts_manager.cc:194] Registered new tserver with Master: 9e41c73815c14d81b0d1cb1c71d8177f (127.24.216.1:33621)
07:46:03.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.225644 25640 ts_manager.cc:194] Registered new tserver with Master: 9e41c73815c14d81b0d1cb1c71d8177f (127.24.216.1:33621)
07:46:03.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.228523 25640 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.216.1:44349
07:46:03.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.229178 25875 heartbeater.cc:347] Connected to a master server at 127.24.216.62:36897
07:46:03.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.229513 25875 heartbeater.cc:464] Registering TS with master...
07:46:03.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.230283 25875 heartbeater.cc:511] Master 127.24.216.62:36897 requested a full tablet report, sending...
07:46:03.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.233146 25485 ts_manager.cc:194] Registered new tserver with Master: 9e41c73815c14d81b0d1cb1c71d8177f (127.24.216.1:33621)
07:46:03.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:03.527336 25883 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:03.527983 25883 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:03.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:03.528755 25883 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:03.564539 25883 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:03.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:03.565770 25883 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.216.2
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.603631 25883 tablet_server_runner.cc:78] Tablet server non-default flags:
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/data
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/wal
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.2:0
07:46:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:03.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/data/info.pb
07:46:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.2
07:46:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:46:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:03.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.216.2
07:46:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/logs
07:46:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.605166 25883 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:03.607290 25883 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:03.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:03.635869 25890 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:04.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:04.233577 25874 heartbeater.cc:503] Master 127.24.216.60:46401 was elected leader, sending a full tablet report...
07:46:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.028921 25889 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 25883
07:46:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.054715 25889 kernel_stack_watchdog.cc:198] Thread 25883 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 390ms:
07:46:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
07:46:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
07:46:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
07:46:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
07:46:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.054831 25883 thread.cc:636] GCE (cloud detector) Time spent creating pthread: real 1.421s user 0.475s sys 0.944s
07:46:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.055205 25883 thread.cc:602] GCE (cloud detector) Time spent starting thread: real 1.422s user 0.475s sys 0.944s
07:46:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:03.636044 25891 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.066196 25883 server_base.cc:1034] running on GCE node
07:46:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.066277 25894 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.067740 25883 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.070008 25883 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.071403 25883 hybrid_clock.cc:648] HybridClock initialized: now 1790408765071337 us; error 65 us; skew 500 ppm
07:46:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.072218 25883 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:05.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.079332 25883 webserver.cc:460] Webserver started at http://127.24.216.2:40733/ using document root <none> and password file <none>
07:46:05.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.080255 25883 fs_manager.cc:359] Metadata directory not provided
07:46:05.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.080461 25883 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.080967 25883 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.086256 25883 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/data/instance:
07:46:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "471b49913fee448fb022634354288fd3"
07:46:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:05 on dist-test-slave-f0nh"
07:46:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.087749 25883 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/wal/instance:
07:46:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "471b49913fee448fb022634354288fd3"
07:46:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:05 on dist-test-slave-f0nh"
07:46:05.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.095320 25883 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
07:46:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.101332 25900 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.102428 25883 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.102713 25883 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/wal
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "471b49913fee448fb022634354288fd3"
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:05 on dist-test-slave-f0nh"
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.103005 25883 fs_report.cc:383] FS layout report
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/wal
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/wal
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/data/data
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:05.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:05.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.151412 25883 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:05.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.152908 25883 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:05.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.153321 25883 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:05.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.156211 25883 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:46:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.162724 25883 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:46:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.163007 25883 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:46:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.163353 25883 ts_tablet_manager.cc:610] Registered 0 tablets
07:46:05.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.163589 25883 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:46:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.312840 25883 rpc_server.cc:307] RPC server started. Bound to: 127.24.216.2:43487
07:46:05.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.312925 26013 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.216.2:43487 every 8 connection(s)
07:46:05.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.315503 25883 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/data/info.pb
07:46:05.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.326407 25440 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 25883
07:46:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.326974 25440 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-1/wal/instance
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.347746 25440 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/wal
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/data
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/logs
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/data/info.pb
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:05.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.3:0
07:46:05.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.216.3
07:46:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.3
07:46:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:46:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:46:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:05.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
07:46:05.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.372359 26014 heartbeater.cc:347] Connected to a master server at 127.24.216.61:41609
07:46:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.372437 26015 heartbeater.cc:347] Connected to a master server at 127.24.216.60:46401
07:46:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.372895 26014 heartbeater.cc:464] Registering TS with master...
07:46:05.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.372972 26015 heartbeater.cc:464] Registering TS with master...
07:46:05.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.374441 26014 heartbeater.cc:511] Master 127.24.216.61:41609 requested a full tablet report, sending...
07:46:05.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.374552 26015 heartbeater.cc:511] Master 127.24.216.60:46401 requested a full tablet report, sending...
07:46:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.377772 25640 ts_manager.cc:194] Registered new tserver with Master: 471b49913fee448fb022634354288fd3 (127.24.216.2:43487)
07:46:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.378206 25560 ts_manager.cc:194] Registered new tserver with Master: 471b49913fee448fb022634354288fd3 (127.24.216.2:43487)
07:46:05.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.379755 25640 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.216.2:42467
07:46:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.381820 26016 heartbeater.cc:347] Connected to a master server at 127.24.216.62:36897
07:46:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.382182 26016 heartbeater.cc:464] Registering TS with master...
07:46:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.383140 26016 heartbeater.cc:511] Master 127.24.216.62:36897 requested a full tablet report, sending...
07:46:05.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.385864 25485 ts_manager.cc:194] Registered new tserver with Master: 471b49913fee448fb022634354288fd3 (127.24.216.2:43487)
07:46:05.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.665750 26024 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:05.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.666204 26024 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:05.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.666730 26024 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:05.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.698472 26024 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:05.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.699246 26024 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.24.216.3
07:46:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.733302 26024 tablet_server_runner.cc:78] Tablet server non-default flags:
07:46:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.24.216.20:39873
07:46:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:05.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/data
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/wal
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.24.216.3:0
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/data/info.pb
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.24.216.3
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.24.216.62:36897,127.24.216.61:41609,127.24.216.60:46401
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.24.216.3
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/logs
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:05.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:05.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.734802 26024 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:05.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:05.736671 26024 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.756440 26031 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:06.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:06.385128 26015 heartbeater.cc:503] Master 127.24.216.60:46401 was elected leader, sending a full tablet report...
07:46:06.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.756798 26033 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:06.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:06.995000 26032 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:46:06.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:05.756459 26030 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:06.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:06.995255 26024 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:46:07.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.000265 26024 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:07.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.002597 26024 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.003932 26024 hybrid_clock.cc:648] HybridClock initialized: now 1790408767003894 us; error 54 us; skew 500 ppm
07:46:07.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.004707 26024 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:07.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.011168 26024 webserver.cc:460] Webserver started at http://127.24.216.3:44651/ using document root <none> and password file <none>
07:46:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.012055 26024 fs_manager.cc:359] Metadata directory not provided
07:46:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.012240 26024 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.012619 26024 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.017702 26024 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/data/instance:
07:46:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ca0618376b64cc49bb83b8c048209bf"
07:46:07.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:07 on dist-test-slave-f0nh"
07:46:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.018788 26024 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/wal/instance:
07:46:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ca0618376b64cc49bb83b8c048209bf"
07:46:07.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:07 on dist-test-slave-f0nh"
07:46:07.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.026044 26024 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
07:46:07.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.031620 26040 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.032836 26024 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.033157 26024 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/wal
07:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5ca0618376b64cc49bb83b8c048209bf"
07:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:07 on dist-test-slave-f0nh"
07:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.033499 26024 fs_report.cc:383] FS layout report
07:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/wal
07:46:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/wal
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/data/data
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:07.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:07.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.094035 26024 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:07.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.095494 26024 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:07.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.096026 26024 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:07.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.098608 26024 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
07:46:07.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.102898 26024 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
07:46:07.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.103113 26024 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
07:46:07.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.103303 26024 ts_tablet_manager.cc:610] Registered 0 tablets
07:46:07.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.103434 26024 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
07:46:07.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.245299 26024 rpc_server.cc:307] RPC server started. Bound to: 127.24.216.3:41479
07:46:07.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.245395 26153 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.24.216.3:41479 every 8 connection(s)
07:46:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.247958 26024 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/data/info.pb
07:46:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.257718 25440 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 26024
07:46:07.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.258252 25440 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8052316012689429502/ts-2/wal/instance
07:46:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.294006 26155 heartbeater.cc:347] Connected to a master server at 127.24.216.60:46401
07:46:07.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.294490 26155 heartbeater.cc:464] Registering TS with master...
07:46:07.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.295079 26154 heartbeater.cc:347] Connected to a master server at 127.24.216.61:41609
07:46:07.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.295419 26154 heartbeater.cc:464] Registering TS with master...
07:46:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.295722 26155 heartbeater.cc:511] Master 127.24.216.60:46401 requested a full tablet report, sending...
07:46:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.296456 26154 heartbeater.cc:511] Master 127.24.216.61:41609 requested a full tablet report, sending...
07:46:07.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.298633 25640 ts_manager.cc:194] Registered new tserver with Master: 5ca0618376b64cc49bb83b8c048209bf (127.24.216.3:41479)
07:46:07.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.299856 25560 ts_manager.cc:194] Registered new tserver with Master: 5ca0618376b64cc49bb83b8c048209bf (127.24.216.3:41479)
07:46:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.300750 26156 heartbeater.cc:347] Connected to a master server at 127.24.216.62:36897
07:46:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.300863 25640 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.24.216.3:55489
07:46:07.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.301098 26156 heartbeater.cc:464] Registering TS with master...
07:46:07.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.301899 26156 heartbeater.cc:511] Master 127.24.216.62:36897 requested a full tablet report, sending...
07:46:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.304014 25485 ts_manager.cc:194] Registered new tserver with Master: 5ca0618376b64cc49bb83b8c048209bf (127.24.216.3:41479)
07:46:07.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.309435 25440 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
07:46:07.320 [DEBUG - main] (MiniKuduCluster.java:195) Response:
07:46:07.321 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
07:46:07.332 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.24.216.62"
port: 36897
}
bound_http_address {
host: "127.24.216.62"
port: 44023
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.24.216.61"
port: 41609
}
bound_http_address {
host: "127.24.216.61"
port: 44549
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.24.216.60"
port: 46401
}
bound_http_address {
host: "127.24.216.60"
port: 37961
}
}
}
07:46:07.337 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:46:07.339 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.24.216.1"
port: 33621
}
bound_http_address {
host: "127.24.216.1"
port: 37185
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.24.216.2"
port: 43487
}
bound_http_address {
host: "127.24.216.2"
port: 40733
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.24.216.3"
port: 41479
}
bound_http_address {
host: "127.24.216.3"
port: 44651
}
}
}
07:46:07.339 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
07:46:07.511 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
07:46:07.628 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskHOPlnL/test-tmp
07:46:07.629 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
07:46:07.630 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskHOPlnL/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
07:46:07.637 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
07:46:07.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:07.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
07:46:07.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.980916 26165 mini_oidc.cc:152] Starting JWKS server
07:46:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.984794 26165 webserver.cc:460] Webserver started at http://127.0.0.1:35103/ using document root <none> and password file <none>
07:46:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.985312 26165 mini_oidc.cc:189] Starting OIDC Discovery server
07:46:07.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:07.987064 26165 webserver.cc:460] Webserver started at http://127.0.0.1:38209/ using document root <none> and password file <none>
07:46:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
07:46:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/principal' for realm 'KRBTEST.COM',
07:46:08.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
07:46:08.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:08 dist-test-slave-f0nh krb5kdc[26177](info): setting up network...
07:46:08.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
07:46:08.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:08 dist-test-slave-f0nh krb5kdc[26177](info): set up 2 sockets
07:46:08.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:08 dist-test-slave-f0nh krb5kdc[26177](info): commencing operation
07:46:08.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
07:46:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:08.306159 26155 heartbeater.cc:503] Master 127.24.216.60:46401 was elected leader, sending a full tablet report...
07:46:09.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.090711 26165 mini_kdc.cc:121] Time spent starting KDC: real 1.079s user 0.006s sys 0.013s
07:46:09.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
07:46:09.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:46:09.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
07:46:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
07:46:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:46:09.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
07:46:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
07:46:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:46:09.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
07:46:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
07:46:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/test-user.keytab.
07:46:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/test-user.keytab.
07:46:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:09 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408769, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:46:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T07:46:09Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
07:46:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T07:46:09Z Disabled control of system clock
07:46:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.141.126@KRBTEST.COM; defaulting to no policy
07:46:09.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:09.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.141.126@KRBTEST.COM" created.
07:46:09.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:09.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.141.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.126.keytab.
07:46:09.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.141.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.126.keytab.
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:09.293105 26165 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/logs
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/info.pb
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:09.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.126:46865
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.126
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.126.keytab
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.126
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.619154 26190 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.619731 26190 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:09.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.620313 26190 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.650928 26190 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.651147 26190 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.651400 26190 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.651592 26190 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.651777 26190 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.651965 26190 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.652104 26190 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35103/jwks.json
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:09.689206 26190 master_runner.cc:386] Master server non-default flags:
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.126.keytab
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.126
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:09.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.126:46865
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/info.pb
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.126
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/logs
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:09.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:09.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:09.690402 26190 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:09.691763 26190 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:09.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.711377 26196 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:10.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.711410 26199 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:09.712195 26197 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:10.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:09.713527 26190 server_base.cc:1034] running on GCE node
07:46:10.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.968307 26190 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:10.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.971347 26190 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.972803 26190 hybrid_clock.cc:648] HybridClock initialized: now 1790408770972759 us; error 60 us; skew 500 ppm
07:46:10.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.973589 26190 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:10 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408770, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:10.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.980541 26190 init.cc:377] Logged in from keytab as kudu/127.25.141.126@KRBTEST.COM (short username kudu)
07:46:10.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.994446 26190 webserver.cc:460] Webserver started at http://127.25.141.126:33851/ using document root <none> and password file <none>
07:46:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.996075 26190 fs_manager.cc:359] Metadata directory not provided
07:46:10.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.996588 26190 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:10.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:10.997305 26190 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.005574 26190 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/instance:
07:46:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "43914629839946289471817dd735c3c0"
07:46:11.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:10 on dist-test-slave-f0nh"
07:46:11.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.007020 26190 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal/instance:
07:46:11.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "43914629839946289471817dd735c3c0"
07:46:11.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:10 on dist-test-slave-f0nh"
07:46:11.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.016553 26190 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.003s sys 0.004s
07:46:11.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.024636 26208 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:11.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.026386 26190 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.026801 26190 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "43914629839946289471817dd735c3c0"
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:10 on dist-test-slave-f0nh"
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.027215 26190 fs_report.cc:383] FS layout report
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/data
07:46:11.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:11.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.067585 26190 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:11.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.073103 26190 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:11.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.073599 26190 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.143039 26190 rpc_server.cc:307] RPC server started. Bound to: 127.25.141.126:46865
07:46:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.143131 26259 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.141.126:46865 every 8 connection(s)
07:46:11.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.145694 26190 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/info.pb
07:46:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.146894 26165 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 26190
07:46:11.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.147387 26165 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal/instance
07:46:11.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.153379 26260 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:11.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.141.125@KRBTEST.COM; defaulting to no policy
07:46:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:11.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.141.125@KRBTEST.COM" created.
07:46:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.173636 26260 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } has no permanent_uuid. Determining permanent_uuid...
07:46:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.141.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.125.keytab.
07:46:11.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.141.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.125.keytab.
07:46:11.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:11 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408770, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.126@KRBTEST.COM for kudu/127.25.141.126@KRBTEST.COM
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.199786 26165 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/logs
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/info.pb
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.125:44005
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.125
07:46:11.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.125.keytab
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.125
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:11.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:11.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:11.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.213884 26260 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } has no permanent_uuid. Determining permanent_uuid...
07:46:11.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.216364 26210 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.220415 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.125:44005: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:11.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.269956 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } attempt: 1
07:46:11.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.274024 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.125:44005: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:11.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.342450 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } attempt: 2
07:46:11.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.345897 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.125:44005: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:11.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.437259 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } attempt: 3
07:46:11.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.441800 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.125:44005: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.536118 26266 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:11.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.536705 26266 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:11.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.537348 26266 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:11.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.570400 26266 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:11.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.570623 26266 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:11.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.570885 26266 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:11.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.571076 26266 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:11.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.571259 26266 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:11.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.571449 26266 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:11.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.571611 26266 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35103/jwks.json
07:46:11.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.585222 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } attempt: 4
07:46:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.588826 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.125:44005: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.609378 26266 master_runner.cc:386] Master server non-default flags:
07:46:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:11.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.125.keytab
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.125
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.125:44005
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:11.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/info.pb
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.125
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/logs
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:11.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.610558 26266 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:11.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.611485 26266 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:11.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.631568 26274 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:11.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.888217 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } attempt: 5
07:46:11.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.892987 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.125:44005: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:12.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.440371 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } attempt: 6
07:46:12.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:12.447089 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.125:44005: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:11.633137 26266 server_base.cc:1034] running on GCE node
07:46:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.633641 26273 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:11.631930 26276 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.894575 26266 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:12.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.897430 26266 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.898797 26266 hybrid_clock.cc:648] HybridClock initialized: now 1790408772898764 us; error 36 us; skew 500 ppm
07:46:12.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.899649 26266 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:12.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:12 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408772, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.905486 26266 init.cc:377] Logged in from keytab as kudu/127.25.141.125@KRBTEST.COM (short username kudu)
07:46:12.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.911151 26266 webserver.cc:460] Webserver started at http://127.25.141.125:41461/ using document root <none> and password file <none>
07:46:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.912163 26266 fs_manager.cc:359] Metadata directory not provided
07:46:12.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.912423 26266 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:12.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.912997 26266 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:12.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.918583 26266 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/instance:
07:46:12.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "045026620edd4e31bddc42415a52a719"
07:46:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:12 on dist-test-slave-f0nh"
07:46:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.920332 26266 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal/instance:
07:46:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "045026620edd4e31bddc42415a52a719"
07:46:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:12 on dist-test-slave-f0nh"
07:46:12.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.929721 26266 fs_manager.cc:691] Time spent creating directory manager: real 0.009s user 0.007s sys 0.001s
07:46:12.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.935793 26289 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:12.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.937457 26266 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.937846 26266 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "045026620edd4e31bddc42415a52a719"
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:12 on dist-test-slave-f0nh"
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.938246 26266 fs_report.cc:383] FS layout report
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/data
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:12.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.972337 26266 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.977802 26266 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:12.978274 26266 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.059952 26266 rpc_server.cc:307] RPC server started. Bound to: 127.25.141.125:44005
07:46:13.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.060040 26340 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.141.125:44005 every 8 connection(s)
07:46:13.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.062523 26266 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/info.pb
07:46:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.071605 26165 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 26266
07:46:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.072124 26165 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal/instance
07:46:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.072532 26341 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.25.141.124@KRBTEST.COM; defaulting to no policy
07:46:13.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.088176 26341 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } has no permanent_uuid. Determining permanent_uuid...
07:46:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:13.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.25.141.124@KRBTEST.COM" created.
07:46:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
07:46:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.141.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.124.keytab.
07:46:13.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.25.141.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.124.keytab.
07:46:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:13 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408772, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.125@KRBTEST.COM for kudu/127.25.141.126@KRBTEST.COM
07:46:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.121551 26165 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/logs
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/info.pb
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.124:36675
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.124
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.124.keytab
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.124
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:13.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:13.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.133616 26341 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } has no permanent_uuid. Determining permanent_uuid...
07:46:13.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:13 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408772, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.125@KRBTEST.COM for kudu/127.25.141.125@KRBTEST.COM
07:46:13.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.166962 26341 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } has no permanent_uuid. Determining permanent_uuid...
07:46:13.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.169618 26293 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.174316 26341 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.223790 26341 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 1
07:46:13.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.227594 26341 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.296068 26341 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 2
07:46:13.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.300495 26341 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.391961 26341 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 3
07:46:13.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.395747 26341 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.450685 26347 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.451203 26347 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:13.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.451786 26347 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:13.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.482654 26347 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:13.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.482926 26347 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:13.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.483193 26347 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:13.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.483461 26347 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:13.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.483671 26347 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:13.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.483862 26347 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:13.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.484088 26347 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35103/jwks.json
07:46:13.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.507709 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } attempt: 7
07:46:13.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:13 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408770, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.126@KRBTEST.COM for kudu/127.25.141.125@KRBTEST.COM
07:46:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.527347 26347 master_runner.cc:386] Master server non-default flags:
07:46:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:13.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.124.keytab
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.124
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.124:36675
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/info.pb
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.124
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/logs
07:46:13.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:13.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.528674 26347 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:13.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.530010 26347 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:13.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.531791 26260 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } has no permanent_uuid. Determining permanent_uuid...
07:46:13.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.536489 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.539173 26341 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 4
07:46:13.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.543910 26341 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.550176 26362 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:13.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.594892 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 1
07:46:13.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.600323 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.681910 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 2
07:46:13.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.687300 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.772750 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 3
07:46:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.776517 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.843348 26341 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 5
07:46:13.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.848678 26341 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:13.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:13.917129 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 4
07:46:13.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.921540 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.205119 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 5
07:46:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:14.209926 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:14.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.396117 26341 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 6
07:46:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:14.402717 26341 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:14.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.762301 26260 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } attempt: 6
07:46:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:14.768766 26260 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.25.141.124:36675: Network error: Client connection negotiation failed: client connection to 127.25.141.124:36675: connect: Connection refused (error 111)
07:46:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.550184 26360 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:13.550561 26359 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:14.784044 26361 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
07:46:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.784210 26347 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
07:46:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.788326 26347 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.790870 26347 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:14.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.792254 26347 hybrid_clock.cc:648] HybridClock initialized: now 1790408774792213 us; error 41 us; skew 500 ppm
07:46:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.793188 26347 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:14 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408774, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.799118 26347 init.cc:377] Logged in from keytab as kudu/127.25.141.124@KRBTEST.COM (short username kudu)
07:46:14.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.804193 26347 webserver.cc:460] Webserver started at http://127.25.141.124:39729/ using document root <none> and password file <none>
07:46:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.805186 26347 fs_manager.cc:359] Metadata directory not provided
07:46:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.805418 26347 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:14.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.805974 26347 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
07:46:14.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.810876 26347 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/instance:
07:46:14.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f6d96cb5b864271871f2add8156ae8e"
07:46:14.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:14 on dist-test-slave-f0nh"
07:46:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.812021 26347 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal/instance:
07:46:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f6d96cb5b864271871f2add8156ae8e"
07:46:14.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:14 on dist-test-slave-f0nh"
07:46:14.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.819078 26347 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
07:46:14.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.824688 26374 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:14.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.825727 26347 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
07:46:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.826081 26347 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f6d96cb5b864271871f2add8156ae8e"
07:46:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:14 on dist-test-slave-f0nh"
07:46:14.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.826419 26347 fs_report.cc:383] FS layout report
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/data
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:14.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:14.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.869915 26347 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.875245 26347 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:14.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.875653 26347 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:14.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.950793 26347 rpc_server.cc:307] RPC server started. Bound to: 127.25.141.124:36675
07:46:14.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.950850 26425 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.141.124:36675 every 8 connection(s)
07:46:14.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.953538 26347 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/info.pb
07:46:14.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.959110 26426 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
07:46:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.962389 26165 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 26347
07:46:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.962726 26165 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal/instance
07:46:14.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:14.972069 26426 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } has no permanent_uuid. Determining permanent_uuid...
07:46:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:14 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408774, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.124@KRBTEST.COM for kudu/127.25.141.126@KRBTEST.COM
07:46:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:14 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408769, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.141.124@KRBTEST.COM
07:46:15.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.004855 26426 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } has no permanent_uuid. Determining permanent_uuid...
07:46:15.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:15 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408774, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.124@KRBTEST.COM for kudu/127.25.141.125@KRBTEST.COM
07:46:15.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:15 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408769, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.141.125@KRBTEST.COM
07:46:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.029752 26426 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } has no permanent_uuid. Determining permanent_uuid...
07:46:15.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:15 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408774, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.124@KRBTEST.COM for kudu/127.25.141.124@KRBTEST.COM
07:46:15.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:15 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408769, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.141.126@KRBTEST.COM
07:46:15.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.061054 26426 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Bootstrap starting.
07:46:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.067842 26165 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
07:46:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.068145 26426 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Neither blocks nor log segments found. Creating new log.
07:46:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.070377 26426 log.cc:824] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Log is configured to *not* fsync() on all Append() calls
07:46:15.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.075757 26426 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: No bootstrap required, opened a new log
07:46:15.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:15.077 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
07:46:15.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
07:46:15.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
07:46:15.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.141.126"
port: 46865
}
bound_http_address {
host: "127.25.141.126"
port: 33851
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.141.125"
port: 44005
}
bound_http_address {
host: "127.25.141.125"
port: 41461
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.141.124"
port: 36675
}
bound_http_address {
host: "127.25.141.124"
port: 39729
}
}
}
07:46:15.095 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
07:46:15.096 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:15.096 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:46:15.097 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:46:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.103945 26426 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:15.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.104561 26426 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:46:15.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.104807 26426 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4f6d96cb5b864271871f2add8156ae8e, State: Initialized, Role: FOLLOWER
07:46:15.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.105484 26426 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:15.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.108385 26436 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } } }
07:46:15.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.109289 26436 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [sys.catalog]: This master's current role is: FOLLOWER
07:46:15.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.110103 26426 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [sys.catalog]: configured and running, proceeding with master startup.
07:46:15.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:15 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408775, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:15.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:46:15.129 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:15.131 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc
07:46:15.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.130879 26380 tablet.cc:2369] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:46:15.138 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:46:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.141366 26448 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:15.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.141667 26448 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:15.145 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc
07:46:15.146 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:46:15.151 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
07:46:15.153 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
07:46:15.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
07:46:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.135023 26347 master_runner.cc:186] Error getting master registration for 127.25.141.126:46865: OK, instance_id { permanent_uuid: "43914629839946289471817dd735c3c0" instance_seqno: 1790408771124002 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
07:46:15.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.168164 26347 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:15 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408775, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
07:46:15.194 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:15.194 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc
07:46:15.196 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
07:46:15.196 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.141.125:44005
07:46:15.198 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
07:46:15.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.199198 26165 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 26266
07:46:15.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.219700 26436 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:15.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.220297 26436 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:15.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.225054 26436 leader_election.cc:288] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43914629839946289471817dd735c3c0 (127.25.141.126:46865), 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005)
07:46:15.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.229880 26376 connection.cc:535] client connection to 127.25.141.125:44005 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:46:15.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.230317 26376 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
07:46:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.234516 26376 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005): Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:15.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:15.238 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.141.126:46865
07:46:15.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
07:46:15.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.238816 26165 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 26190
07:46:15.270 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:15.270 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.25.141.124:36675
07:46:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.269094 26378 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 43914629839946289471817dd735c3c0 (127.25.141.126:46865): Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111)
07:46:15.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.269546 26378 leader_election.cc:302] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f6d96cb5b864271871f2add8156ae8e; no voters: 045026620edd4e31bddc42415a52a719, 43914629839946289471817dd735c3c0
07:46:15.270 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
07:46:15.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.270630 26436 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:15.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.270845 26165 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 26347
07:46:15.291 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:15.291 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.141.125:44005
07:46:15.293 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
07:46:15.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.295457 26165 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:15.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:15.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/logs
07:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/info.pb
07:46:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.125:44005
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.125
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41461
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.125.keytab
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.125
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:15.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.603411 26454 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:15.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.604174 26454 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:15.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.605175 26454 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:15.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.647373 26454 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:15.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.647642 26454 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:15.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.648048 26454 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:15.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.648378 26454 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:15.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.648697 26454 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.649077 26454 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.649307 26454 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35103/jwks.json
07:46:15.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.686580 26454 master_runner.cc:386] Master server non-default flags:
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.125.keytab
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.125
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.125:44005
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:15.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/info.pb
07:46:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.125
07:46:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41461
07:46:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:15.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/logs
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.688066 26454 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:15.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.689531 26454 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:15.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.706311 26461 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.706507 26462 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:15.709244 26454 server_base.cc:1034] running on GCE node
07:46:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:15.706867 26464 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:16.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.971678 26454 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:16.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.974864 26454 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:16.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.976301 26454 hybrid_clock.cc:648] HybridClock initialized: now 1790408776976250 us; error 60 us; skew 500 ppm
07:46:16.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.977164 26454 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:16.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:16 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408776, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:16.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.983053 26454 init.cc:377] Logged in from keytab as kudu/127.25.141.125@KRBTEST.COM (short username kudu)
07:46:16.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.988376 26454 webserver.cc:460] Webserver started at http://127.25.141.125:41461/ using document root <none> and password file <none>
07:46:16.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.989279 26454 fs_manager.cc:359] Metadata directory not provided
07:46:16.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.989478 26454 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:16.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:16.997339 26454 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.007s sys 0.001s
07:46:17.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.001499 26472 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:17.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.002753 26454 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
07:46:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.003101 26454 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "045026620edd4e31bddc42415a52a719"
07:46:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:12 on dist-test-slave-f0nh"
07:46:17.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.005528 26454 fs_report.cc:383] FS layout report
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/data
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:17.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:17.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.063027 26454 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.071039 26454 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.071450 26454 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:17.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.141489 26454 rpc_server.cc:307] RPC server started. Bound to: 127.25.141.125:44005
07:46:17.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.141625 26524 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.141.125:44005 every 8 connection(s)
07:46:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.144119 26454 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/data/info.pb
07:46:17.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.146589 26165 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 26454
07:46:17.147 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:17.147 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.141.126:46865
07:46:17.147 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
07:46:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.149272 26165 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data
07:46:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:17.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/logs
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/info.pb
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.126:46865
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.126
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33851
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.126.keytab
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.126
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:17.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.160619 26525 sys_catalog.cc:263] Verifying existing consensus state
07:46:17.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 07:46:17.161880 26525 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:46:17.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.163970 26454 master.cc:537] Master@127.25.141.125:44005 shutting down...
07:46:17.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.229467 26454 master.cc:552] Master@127.25.141.125:44005 shutdown complete.
07:46:17.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:46:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.475579 26527 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.476116 26527 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:17.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.476696 26527 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.508490 26527 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.508728 26527 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.509064 26527 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.509301 26527 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.509514 26527 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.509696 26527 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.509864 26527 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35103/jwks.json
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.546574 26527 master_runner.cc:386] Master server non-default flags:
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.126.keytab
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.126
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.126:46865
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/info.pb
07:46:17.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.126
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33851
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/logs
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.547695 26527 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:17.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.549073 26527 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.566957 26534 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:18.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.567662 26533 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:17.568313 26527 server_base.cc:1034] running on GCE node
07:46:18.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:17.568645 26536 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:18.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.813591 26527 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:18.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.816212 26527 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:18.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.817631 26527 hybrid_clock.cc:648] HybridClock initialized: now 1790408778817616 us; error 55 us; skew 500 ppm
07:46:18.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.818435 26527 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:18.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:18 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408778, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.827430 26527 init.cc:377] Logged in from keytab as kudu/127.25.141.126@KRBTEST.COM (short username kudu)
07:46:18.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.833170 26527 webserver.cc:460] Webserver started at http://127.25.141.126:33851/ using document root <none> and password file <none>
07:46:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.834288 26527 fs_manager.cc:359] Metadata directory not provided
07:46:18.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.834549 26527 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:18.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.844702 26527 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.002s
07:46:18.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.849833 26545 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:18.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.851121 26527 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
07:46:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.851523 26527 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "43914629839946289471817dd735c3c0"
07:46:18.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:10 on dist-test-slave-f0nh"
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.854725 26527 fs_report.cc:383] FS layout report
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/data
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:18.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.902035 26527 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.907390 26527 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.907781 26527 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:18.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.980018 26527 rpc_server.cc:307] RPC server started. Bound to: 127.25.141.126:46865
07:46:18.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.980137 26596 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.141.126:46865 every 8 connection(s)
07:46:18.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.982748 26527 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/data/info.pb
07:46:18.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.992610 26597 sys_catalog.cc:263] Verifying existing consensus state
07:46:18.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.992511 26165 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 26527
07:46:18.993 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:18.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 07:46:18.993520 26597 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:46:18.994 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.25.141.124:36675
07:46:18.994 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
07:46:18.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.994174 26527 master.cc:537] Master@127.25.141.126:46865 shutting down...
07:46:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:18.996171 26165 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:19.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/logs
07:46:19.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/info.pb
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.124:36675
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.124
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39729
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.124.keytab
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.124
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:19.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
07:46:19.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:19.056844 26527 master.cc:552] Master@127.25.141.126:46865 shutdown complete.
07:46:19.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
07:46:19.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.316280 26599 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
07:46:19.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.317050 26599 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
07:46:19.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.317929 26599 flags.cc:425] Enabled unsafe flag: --never_fsync=true
07:46:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.354462 26599 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
07:46:19.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.354697 26599 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
07:46:19.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.354978 26599 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
07:46:19.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.355180 26599 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
07:46:19.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.355367 26599 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
07:46:19.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.355551 26599 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
07:46:19.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.355690 26599 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35103/jwks.json
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:19.392514 26599 master_runner.cc:386] Master server non-default flags:
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.141.84:34083
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.141.126:46865,127.25.141.125:44005,127.25.141.124:36675
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/krb5kdc/kudu_127.25.141.124.keytab
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.25.141.124
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
07:46:19.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.141.124:36675
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35103/jwks.json
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/info.pb
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.141.124
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39729
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/logs
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 06:55:20 UTC on ccycloud.quasar-qwhbmo.root.comops.site
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4673737
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:19.393776 26599 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
07:46:19.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:19.394729 26599 file_cache.cc:493] Constructed file cache file cache with capacity 419430
07:46:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.412811 26605 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:20.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.412889 26606 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:20.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:19.414554 26599 server_base.cc:1034] running on GCE node
07:46:20.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:19.412902 26608 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
07:46:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.630366 26599 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
07:46:20.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.633041 26599 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
07:46:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.634442 26599 hybrid_clock.cc:648] HybridClock initialized: now 1790408780634416 us; error 38 us; skew 500 ppm
07:46:20.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.635452 26599 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
07:46:20.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:20 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408780, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:20.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.642036 26599 init.cc:377] Logged in from keytab as kudu/127.25.141.124@KRBTEST.COM (short username kudu)
07:46:20.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.646581 26599 webserver.cc:460] Webserver started at http://127.25.141.124:39729/ using document root <none> and password file <none>
07:46:20.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.647439 26599 fs_manager.cc:359] Metadata directory not provided
07:46:20.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.647629 26599 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
07:46:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.655561 26599 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
07:46:20.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.660184 26617 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
07:46:20.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.661497 26599 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
07:46:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.661782 26599 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data,/tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4f6d96cb5b864271871f2add8156ae8e"
07:46:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 07:46:14 on dist-test-slave-f0nh"
07:46:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.664003 26599 fs_report.cc:383] FS layout report
07:46:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
07:46:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/wal
07:46:20.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/data
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
07:46:20.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
07:46:20.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.700080 26599 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
07:46:20.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.705904 26599 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
07:46:20.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.706305 26599 kserver.cc:163] Server-wide thread pool size limit: 3276
07:46:20.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.784654 26599 rpc_server.cc:307] RPC server started. Bound to: 127.25.141.124:36675
07:46:20.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.784785 26668 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.25.141.124:36675 every 8 connection(s)
07:46:20.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.787516 26599 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskHOPlnL/test-tmp/mini-kudu-cluster8226118285756509941/master-2/data/info.pb
07:46:20.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.788081 26165 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu as pid 26599
07:46:20.788 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
07:46:20.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.799356 26669 sys_catalog.cc:263] Verifying existing consensus state
07:46:20.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.806221 26669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Bootstrap starting.
07:46:20.839 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:20.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.853014 26669 log.cc:824] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Log is configured to *not* fsync() on all Append() calls
07:46:20.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.858358 26669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
07:46:20.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.859293 26669 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Bootstrap complete.
07:46:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.895176 26669 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.898736 26669 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
07:46:20.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.899044 26669 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4f6d96cb5b864271871f2add8156ae8e, State: Initialized, Role: FOLLOWER
07:46:20.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.900061 26669 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:20.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.903036 26673 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } } }
07:46:20.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.903895 26673 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [sys.catalog]: This master's current role is: FOLLOWER
07:46:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.904893 26669 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [sys.catalog]: configured and running, proceeding with master startup.
07:46:20.908 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:20.910 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.25.141.124:36675(127.25.141.124:36675)] connecting to peer
07:46:20.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:20.937566 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:20.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:20.937866 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:20.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:20.961344 26623 tablet.cc:2369] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: Can't schedule compaction. Clean time has not been advanced past its initial value.
07:46:20.971 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:20.972 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:20.973 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:20.974 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:20.975 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:20.977 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.25.141.124:36675(127.25.141.124:36675)] Successfully connected to peer
07:46:20.980 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:20.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.997400 26673 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:20.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.997931 26673 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:21.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:20.946657 26599 master_runner.cc:186] Error getting master registration for 127.25.141.126:46865: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111),
07:46:21.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:21.000582 26599 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:21.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:21.001128 26673 leader_election.cc:288] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43914629839946289471817dd735c3c0 (127.25.141.126:46865), 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005)
07:46:21.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:21.003952 26619 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:21.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:21.015523 26619 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005): Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:21.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:21.020886 26621 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 43914629839946289471817dd735c3c0 (127.25.141.126:46865): Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111)
07:46:21.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:21.021328 26621 leader_election.cc:302] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f6d96cb5b864271871f2add8156ae8e; no voters: 045026620edd4e31bddc42415a52a719, 43914629839946289471817dd735c3c0
07:46:21.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:21.022385 26673 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:21.085 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
07:46:21.094 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.25.141.124
07:46:21.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:21 dist-test-slave-f0nh krb5kdc[26177](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408775, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.25.141.124@KRBTEST.COM
07:46:21.356 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x015319c6, L:/127.0.0.1:43372 - R:/127.25.141.124:36675] using SASL/GSSAPI
07:46:21.376 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.395 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:21.397 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:21.399 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.399 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.400 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.412 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.416 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.434 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:21.437 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.440 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:21.440 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.446 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.446 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.446 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.448 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.474 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:21.475 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:21.477 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.477 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.478 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.482 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.482 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.483 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.483 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:21.495 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:21.496 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.496 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.497 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.501 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.534 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:21.535 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:21.537 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.537 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.539 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.539 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.540 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.543 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.614 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:21.615 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:21.618 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.618 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.619 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.621 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.621 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.621 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.623 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.674 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:21.676 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:21.676 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.677 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.677 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.679 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.679 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.680 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.682 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.754 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:21.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:21.756 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.756 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.757 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:21.759 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.759 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.760 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:21.762 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:21.942140 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:21.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:21.942458 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:22.134 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:22.136 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:22.137 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.137 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.139 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.139 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.141 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.142 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.144 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:22.194 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:22.195 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:22.196 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.196 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.197 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.198 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.198 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.199 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.201 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:22.423933 25671 debug-util.cc:398] Leaking SignalData structure 0x7b0800090dc0 after lost signal to thread 25602
07:46:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:22.424654 25671 debug-util.cc:398] Leaking SignalData structure 0x7b0800090d40 after lost signal to thread 25674
07:46:22.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:22.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:22.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.761 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:22.763 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.763 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.763 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:22.763 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:22.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:22.945377 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:22.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:22.945631 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:22.975726 26693 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:22.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:22.976148 26693 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:22.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:22.977804 26693 leader_election.cc:288] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43914629839946289471817dd735c3c0 (127.25.141.126:46865), 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005)
07:46:22.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:22.984050 26621 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 43914629839946289471817dd735c3c0 (127.25.141.126:46865): Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111)
07:46:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:22.984645 26619 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005): Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:22.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:22.985138 26619 leader_election.cc:302] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f6d96cb5b864271871f2add8156ae8e; no voters: 045026620edd4e31bddc42415a52a719, 43914629839946289471817dd735c3c0
07:46:22.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:22.985852 26693 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:23.948318 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:23.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:23.948562 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:24.951310 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:24.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:24.951566 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:25.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:25.954356 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:25.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:25.954610 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:25.994 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:25.994 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:25.996 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:25.996 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:25.997 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:25.997 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:25.997 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:25.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:25.999 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:26.002998 26599 master_runner.cc:186] Error getting master registration for 127.25.141.126:46865: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111),
07:46:26.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:26.003317 26599 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:26.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:26.285543 26703 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:26.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:26.285900 26703 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:26.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:26.287349 26703 leader_election.cc:288] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43914629839946289471817dd735c3c0 (127.25.141.126:46865), 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005)
07:46:26.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:26.288422 26621 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111) [suppressed 3 similar messages]
07:46:26.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:26.292033 26621 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 43914629839946289471817dd735c3c0 (127.25.141.126:46865): Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111)
07:46:26.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:26.292495 26619 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005): Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:26.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:26.292917 26619 leader_election.cc:302] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f6d96cb5b864271871f2add8156ae8e; no voters: 045026620edd4e31bddc42415a52a719, 43914629839946289471817dd735c3c0
07:46:26.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:26.293570 26703 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:26.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:26.957301 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:26.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:26.957553 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:27.494 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:27.495 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:27.496 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.497 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.497 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.498 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.498 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.498 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.500 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:27.814 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:27.815 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:27.816 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.817 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.817 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.820 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:27.854 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:27.856 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:27.857 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.857 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.858 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.858 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.858 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:27.858 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:27.861 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:27.960317 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:27.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:27.960569 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:28.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:28.392874 26708 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:28.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:28.393230 26708 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:28.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:28.394788 26708 leader_election.cc:288] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43914629839946289471817dd735c3c0 (127.25.141.126:46865), 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005)
07:46:28.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:28.399199 26619 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005): Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:28.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:28.399578 26621 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 43914629839946289471817dd735c3c0 (127.25.141.126:46865): Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111)
07:46:28.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:28.399914 26621 leader_election.cc:302] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f6d96cb5b864271871f2add8156ae8e; no voters: 045026620edd4e31bddc42415a52a719, 43914629839946289471817dd735c3c0
07:46:28.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:28.400394 26708 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:28.963351 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:28.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:28.963599 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:29.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:29.967109 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:29.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:29.967423 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:30 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408790, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:30.645622 26613 init.cc:295] Successfully reacquired a new kerberos TGT
07:46:30.654 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:30.655 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:30.656 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:30.656 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:30.657 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:30.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:30.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:30.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:30.660 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:30.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:30.867259 26714 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:30.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:30.867707 26714 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:30.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:30.869387 26714 leader_election.cc:288] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43914629839946289471817dd735c3c0 (127.25.141.126:46865), 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005)
07:46:30.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:30.874300 26621 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 43914629839946289471817dd735c3c0 (127.25.141.126:46865): Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111)
07:46:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:30.875272 26619 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005): Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:30.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:30.875545 26619 leader_election.cc:302] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f6d96cb5b864271871f2add8156ae8e; no voters: 045026620edd4e31bddc42415a52a719, 43914629839946289471817dd735c3c0
07:46:30.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:30.876083 26714 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:30.970350 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:30.970616 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:31.005537 26599 master_runner.cc:186] Error getting master registration for 127.25.141.126:46865: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111),
07:46:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:31.006022 26599 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:31.973699 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:31.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:31.974045 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:32.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:32.976969 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:32.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:32.977211 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:33.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:33.980075 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:33.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:33.980324 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:34.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:34.276005 26721 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:34.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:34.276435 26721 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:34.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:34.278163 26721 leader_election.cc:288] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43914629839946289471817dd735c3c0 (127.25.141.126:46865), 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005)
07:46:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:34.280017 26621 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111) [suppressed 5 similar messages]
07:46:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:34.282807 26619 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005): Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:34.283265 26621 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 43914629839946289471817dd735c3c0 (127.25.141.126:46865): Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111)
07:46:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:34.283629 26621 leader_election.cc:302] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f6d96cb5b864271871f2add8156ae8e; no voters: 045026620edd4e31bddc42415a52a719, 43914629839946289471817dd735c3c0
07:46:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:34.284238 26721 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:34.535 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:34.536 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:34.537 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:34.537 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:34.538 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:34.538 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:34.538 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:34.538 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:34.542 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:34.983042 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:34.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:34.983405 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:35.694 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:35.696 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:35.696 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:35.697 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:35.697 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:35.698 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:35.699 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:35.699 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:35.701 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:35.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:35.986456 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:35.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:35.986784 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:36.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:36.008582 26599 master_runner.cc:186] Error getting master registration for 127.25.141.126:46865: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111),
07:46:36.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:36.008999 26599 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:36.989730 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:36.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:36.990012 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:37.154 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:37.155 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:37.156 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:37.156 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:37.156 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:37.157 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:37.157 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:37.158 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:37.160 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:37.992879 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:37.993166 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:38.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:38.995954 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:38.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:38.996239 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:39.999792 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:40.000149 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:40.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:40.197292 25869 debug-util.cc:398] Leaking SignalData structure 0x7b08000af9c0 after lost signal to thread 25741
07:46:40.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:40.198530 25869 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba800 after lost signal to thread 25872
07:46:40.414 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:40.415 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:40.416 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:40.416 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:40.416 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:40.417 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:40.417 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:40.418 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:40.421 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:40.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 07:46:40 dist-test-slave-f0nh krb5kdc[26177](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790408800, etypes {rep=17 tkt=17 ses=17}, kudu/127.25.141.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
07:46:40.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:40.648191 26613 init.cc:295] Successfully reacquired a new kerberos TGT
07:46:41.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:41.003098 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:41.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:41.003377 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:41.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:41.012185 26599 master_runner.cc:186] Error getting master registration for 127.25.141.126:46865: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111),
07:46:41.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:41.012621 26599 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:41.214 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:41.216 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:41.216 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:41.216 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:41.216 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:41.217 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:41.217 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:41.217 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:41.222 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:41.234 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:41.235 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:41.235 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:41.235 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:41.235 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:41.236 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:41.236 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:41.236 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:41.238 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:42.006564 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:42.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:42.006896 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:42.086848 26665 debug-util.cc:398] Leaking SignalData structure 0x7b08000adf40 after lost signal to thread 26600
07:46:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:42.087505 26665 debug-util.cc:398] Leaking SignalData structure 0x7b0800045180 after lost signal to thread 26668
07:46:42.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:42.966044 25869 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba940 after lost signal to thread 25741
07:46:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:42.967149 25869 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba840 after lost signal to thread 25872
07:46:42.994 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:42.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:42.995 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:42.996 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:42.996 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:42.997 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:42.997 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:42.998 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:43.002 [WARN - pool-6-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:43.010908 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:43.011255 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:44.014118 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:44.014389 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:45.014 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:45.015 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:45.015 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:45.015 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:45.016 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:45.017 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:45.017 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:45.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:45.018710 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:45.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:45.019048 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:45.019 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:46.021886 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:46.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:46.022179 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:46.051429 26599 master_runner.cc:186] Error getting master registration for 127.25.141.126:46865: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111),
07:46:46.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:46.051795 26599 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
07:46:46.274 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:46.274 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:46.275 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:46.275 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:46.275 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:46.276 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:46.276 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:46.276 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:46.278 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:47.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:47.024941 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:47.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:47.025198 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:47.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:47.550499 26740 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
07:46:47.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:47.550908 26740 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43914629839946289471817dd735c3c0" member_type: VOTER last_known_addr { host: "127.25.141.126" port: 46865 } } peers { permanent_uuid: "045026620edd4e31bddc42415a52a719" member_type: VOTER last_known_addr { host: "127.25.141.125" port: 44005 } } peers { permanent_uuid: "4f6d96cb5b864271871f2add8156ae8e" member_type: VOTER last_known_addr { host: "127.25.141.124" port: 36675 } }
07:46:47.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:47.552454 26740 leader_election.cc:288] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43914629839946289471817dd735c3c0 (127.25.141.126:46865), 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005)
07:46:47.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:47.554649 26621 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111) [suppressed 1 similar messages]
07:46:47.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:47.557380 26619 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 045026620edd4e31bddc42415a52a719 (127.25.141.125:44005): Network error: Client connection negotiation failed: client connection to 127.25.141.125:44005: connect: Connection refused (error 111)
07:46:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:47.557811 26621 leader_election.cc:334] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 43914629839946289471817dd735c3c0 (127.25.141.126:46865): Network error: Client connection negotiation failed: client connection to 127.25.141.126:46865: connect: Connection refused (error 111)
07:46:47.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:47.558184 26621 leader_election.cc:302] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4f6d96cb5b864271871f2add8156ae8e; no voters: 045026620edd4e31bddc42415a52a719, 43914629839946289471817dd735c3c0
07:46:47.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:47.558840 26740 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
07:46:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:48.028009 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:48.028311 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:49.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:49.031057 26686 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4f6d96cb5b864271871f2add8156ae8e: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
07:46:49.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 07:46:49.031307 26686 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
07:46:49.754 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.125:44005(127.25.141.125:44005)] connecting to peer
07:46:49.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.25.141.126:46865(127.25.141.126:46865)] connecting to peer
07:46:49.755 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:49.756 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.25.141.125:44005(127.25.141.125:44005)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:49.756 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.125:44005: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005
07:46:49.756 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:49.757 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.126:46865(127.25.141.126:46865)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:49.757 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.25.141.126:46865: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
07:46:49.759 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675; will retry
07:46:49.762 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28967), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [177ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [575ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [579ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [603ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [604ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [614ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [618ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [620ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [640ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [645ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [649ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [651ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [652ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [680ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [684ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [687ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [699ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [701ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [703ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [743ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [745ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [821ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [825ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [826ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [880ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [883ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [885ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [885ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [956ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [959ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [962ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [964ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1337ms] refreshing cache from master, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1346ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1347ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1397ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1400ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1957ms] refreshing cache from master, [1957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1964ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1965ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1966ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1966ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [5196ms] refreshing cache from master, [5196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [5197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [5199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [5199ms] too many traces: truncated at 100 traces, deferred=Deferred@1042707631(state=PENDING, result=null, callback=(continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901), errback=(continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901)))
07:46:49.767 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.141.124:36675(127.25.141.124:36675)] disconnecting while in state READY
07:46:49.768 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.141.126:46865(127.25.141.126:46865)] disconnecting while in state TERMINATED
07:46:49.768 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.25.141.125:44005(127.25.141.125:44005)] disconnecting while in state TERMINATED
07:46:49.772 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:46:49.774 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.25.141.124:36675(127.25.141.124:36675)] handling channelInactive
07:46:49.774 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:46:49.775 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.25.141.124:36675(127.25.141.124:36675)] cleaning up while in state READY due to: connection closed
07:46:49.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:49.778182 26165 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 26599
07:46:50.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T07:46:50Z chronyd exiting
07:46:51.987 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
07:46:51.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:51.989136 25440 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 25722
07:46:52.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:52.015445 25440 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 25883
07:46:52.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:52.042063 25440 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 26024
07:46:52.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:52.068423 25440 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 25452
07:46:52.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:52.097955 25440 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 25522
07:46:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 07:46:52.122339 25440 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskHOPlnL/build/tsan/bin/kudu with pid 25597
07:46:53.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T07:46:53Z chronyd exiting
07:46:54.327 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28967), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [177ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [575ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [579ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [603ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [604ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [614ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [618ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [620ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [640ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [645ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [649ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [651ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [652ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [680ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [684ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [687ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [699ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [701ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [703ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [743ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [745ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [821ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [825ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [826ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [880ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [883ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [885ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [885ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [956ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [959ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [962ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [964ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1337ms] refreshing cache from master, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1346ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1347ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1397ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1400ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1957ms] refreshing cache from master, [1957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1964ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1965ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1966ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1966ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [5196ms] refreshing cache from master, [5196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [5197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [5199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [5199ms] too many traces: truncated at 100 traces, deferred=Deferred@1042707631(state=PENDING, result=null, callback=(continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901), errback=(continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.141.126:46865
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=28967), Traces: [0ms] refreshing cache from master, [31ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [111ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [112ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [177ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [176ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [575ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [579ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [603ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [604ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [614ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [618ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [620ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [640ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [643ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [645ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [649ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [651ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [652ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [677ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [679ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [680ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [684ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [685ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [687ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [697ms] refreshing cache from master, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [698ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [699ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [699ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [701ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [703ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [704ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [736ms] refreshing cache from master, [737ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [738ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [739ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [740ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [743ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [745ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [746ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [817ms] refreshing cache from master, [817ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [818ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [819ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [821ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [824ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [825ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [826ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [878ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [880ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [883ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [885ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [885ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [956ms] refreshing cache from master, [957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [959ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [962ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [964ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [965ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1337ms] refreshing cache from master, [1337ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1339ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1340ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1342ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1346ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1347ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1397ms] refreshing cache from master, [1397ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1398ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1400ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1400ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1401ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1403ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1404ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1957ms] refreshing cache from master, [1957ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [1958ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [1959ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [1964ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.125:44005: Network error: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005, [1965ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.124:36675: OK, [1966ms] Sub RPC ConnectToMaster: received response from server master-127.25.141.126:46865: Network error: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [1966ms] delaying RPC due to: Service unavailable: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865, [5196ms] refreshing cache from master, [5196ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.125:44005, [5197ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.126:46865, [5199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.25.141.124:36675, [5199ms] too many traces: truncated at 100 traces, deferred=Deferred@1042707631(state=PENDING, result=null, callback=(continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901), errback=(continuation of Deferred@403535664 after retry RPC after error@1265971176) -> (continuation of Deferred@1802939249 after retry RPC after error@1727539644) -> (continuation of Deferred@364865188 after retry RPC after error@2010583444) -> (continuation of Deferred@690225468 after retry RPC after error@238450797) -> (continuation of Deferred@1624417609 after retry RPC after error@801526646) -> (continuation of Deferred@652380755 after retry RPC after error@455769074) -> (continuation of Deferred@1856470323 after retry RPC after error@1936186994) -> (continuation of Deferred@1097167455 after retry RPC after error@156791914) -> (continuation of Deferred@1556340375 after retry RPC after error@286924595) -> (continuation of Deferred@2064085189 after retry RPC after error@825663723) -> (continuation of Deferred@1942218556 after retry RPC after error@845254128) -> (continuation of Deferred@1923109409 after retry RPC after error@1989514598) -> (continuation of Deferred@436430479 after retry RPC after error@607348274) -> (continuation of Deferred@1416759759 after retry RPC after error@244750466) -> (continuation of Deferred@262185153 after retry RPC after error@807224817) -> (continuation of Deferred@1961168460 after retry RPC after error@1317915354) -> (continuation of Deferred@1151458658 after retry RPC after error@508111562) -> (continuation of Deferred@1107476513 after retry RPC after error@1652099489) -> (continuation of Deferred@679055937 after retry RPC after error@2142364414) -> (continuation of Deferred@736503521 after retry RPC after error@33035837) -> (continuation of Deferred@1702324185 after retry RPC after error@354541467) -> (continuation of Deferred@267006704 after retry RPC after error@1315083470) -> (continuation of Deferred@284523380 after retry RPC after error@196326390) -> (continuation of Deferred@1427144130 after retry RPC after error@260982602) -> (continuation of Deferred@882615060 after retry RPC after error@572627570) -> (continuation of Deferred@1955138969 after retry RPC after error@163618901)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.25.141.125:44005,127.25.141.126:46865,127.25.141.124:36675) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.125:44005(127.25.141.125:44005): Connection refused: /127.25.141.125:44005,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.25.141.126:46865(127.25.141.126:46865): Connection refused: /127.25.141.126:46865
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.25.141.126:46865
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)