15:58:38.653 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:58:38.666 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDtRWV2/test-tmp
15:58:38.668 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:58:38.672 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDtRWV2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:58:38.940 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:58:39.088 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:58:39.089 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:58:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:39.092551 374 mini_oidc.cc:152] Starting JWKS server
15:58:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:39.097050 374 webserver.cc:460] Webserver started at http://127.0.0.1:42183/ using document root <none> and password file <none>
15:58:39.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:39.097541 374 mini_oidc.cc:189] Starting OIDC Discovery server
15:58:39.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:39.099332 374 webserver.cc:460] Webserver started at http://127.0.0.1:38401/ using document root <none> and password file <none>
15:58:39.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T15:58:39Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:58:39.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T15:58:39Z Disabled control of system clock
15:58:39.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:39.152271 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/wal
15:58:39.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/data
15:58:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/logs
15:58:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/data/info.pb
15:58:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:39.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:58:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:37821
15:58:39.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
15:58:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:39.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:39.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42183/jwks.json with env {}
15:58:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.454917 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:39.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.455654 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:39.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.456365 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:39.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.488456 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:58:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.488822 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.489151 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:58:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.489483 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:58:39.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.489866 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:58:39.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.490140 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42183/jwks.json
15:58:39.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:39.525293 386 master_runner.cc:386] Master server non-default flags:
15:58:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/data
15:58:39.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/wal
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:37821
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:39.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42183/jwks.json
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/data/info.pb
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:39.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/logs
15:58:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:58:39.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:39.526876 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:39.529114 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:39.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.550513 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:40.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.550506 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:40.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:39.550592 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:40.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:40.778128 394 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1227 milliseconds
15:58:40.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.778257 386 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:58:40.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.779699 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:58:40.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.782829 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:58:40.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.784277 386 hybrid_clock.cc:648] HybridClock initialized: now 1790438320784232 us; error 65 us; skew 500 ppm
15:58:40.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.785072 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:58:40.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.797152 386 webserver.cc:460] Webserver started at http://127.0.93.190:34913/ using document root <none> and password file <none>
15:58:40.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.798080 386 fs_manager.cc:359] Metadata directory not provided
15:58:40.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.798265 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:58:40.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.798779 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:58:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.804111 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/data/instance:
15:58:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da70e45476d64c0da4573f61137fffdb"
15:58:40.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:40 on dist-test-slave-874w"
15:58:40.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.805554 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/wal/instance:
15:58:40.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da70e45476d64c0da4573f61137fffdb"
15:58:40.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:40 on dist-test-slave-874w"
15:58:40.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.814702 386 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
15:58:40.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.821034 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:58:40.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.822533 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.007s sys 0.000s
15:58:40.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.822845 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/wal
15:58:40.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da70e45476d64c0da4573f61137fffdb"
15:58:40.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:40 on dist-test-slave-874w"
15:58:40.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.823163 386 fs_report.cc:383] FS layout report
15:58:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:58:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/wal
15:58:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/wal
15:58:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/data/data
15:58:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:58:40.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:58:40.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:58:40.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.859519 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:58:40.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.861397 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:58:40.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.862041 386 kserver.cc:163] Server-wide thread pool size limit: 3276
15:58:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.940553 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:37821
15:58:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.940637 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:37821 every 8 connection(s)
15:58:40.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.943527 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/data/info.pb
15:58:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.947374 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 386
15:58:40.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.947899 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-0/wal/instance
15:58:40.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.949971 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:58:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.953074 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/wal
15:58:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/data
15:58:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/logs
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/data/info.pb
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:40.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:33885
15:58:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
15:58:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:40.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:40.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:40.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:40.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42183/jwks.json with env {}
15:58:40.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.965039 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } has no permanent_uuid. Determining permanent_uuid...
15:58:40.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:40.997130 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } has no permanent_uuid. Determining permanent_uuid...
15:58:41.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:40.999604 406 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33885: connect: Connection refused (error 111)
15:58:41.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.004067 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33885: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33885: connect: Connection refused (error 111)
15:58:41.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.053685 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } attempt: 1
15:58:41.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.057957 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33885: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33885: connect: Connection refused (error 111)
15:58:41.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.126493 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } attempt: 2
15:58:41.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.130632 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33885: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33885: connect: Connection refused (error 111)
15:58:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.222112 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } attempt: 3
15:58:41.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.225992 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33885: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33885: connect: Connection refused (error 111)
15:58:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.273731 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:41.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.274282 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:41.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.274914 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:41.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.309240 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:58:41.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.309535 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:41.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.309722 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:58:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.309897 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:58:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.310174 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:58:41.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.310316 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42183/jwks.json
15:58:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.346125 456 master_runner.cc:386] Master server non-default flags:
15:58:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:41.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/data
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/wal
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:33885
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42183/jwks.json
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/data/info.pb
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/logs
15:58:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:41.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:58:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.347457 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.349419 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:41.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.368466 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:41.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.369455 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } attempt: 4
15:58:41.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.373975 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33885: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33885: connect: Connection refused (error 111)
15:58:41.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.673439 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } attempt: 5
15:58:41.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.678231 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33885: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33885: connect: Connection refused (error 111)
15:58:42.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.225870 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } attempt: 6
15:58:42.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:42.233858 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:33885: Network error: Client connection negotiation failed: client connection to 127.0.93.189:33885: connect: Connection refused (error 111)
15:58:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.368468 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:42.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:41.369445 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:42.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:41.373436 456 server_base.cc:1034] running on GCE node
15:58:42.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.594985 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:58:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.598827 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:58:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.600565 456 hybrid_clock.cc:648] HybridClock initialized: now 1790438322600517 us; error 55 us; skew 500 ppm
15:58:42.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.601414 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:58:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.609047 456 webserver.cc:460] Webserver started at http://127.0.93.189:44231/ using document root <none> and password file <none>
15:58:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.609977 456 fs_manager.cc:359] Metadata directory not provided
15:58:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.610160 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:58:42.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.610675 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:58:42.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.615974 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/data/instance:
15:58:42.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "734e7d30b0e74e73a8c50a54dd77451c"
15:58:42.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:42 on dist-test-slave-874w"
15:58:42.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.617225 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/wal/instance:
15:58:42.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "734e7d30b0e74e73a8c50a54dd77451c"
15:58:42.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:42 on dist-test-slave-874w"
15:58:42.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.625157 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.004s
15:58:42.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.631562 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:58:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.632830 456 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
15:58:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.633206 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/wal
15:58:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "734e7d30b0e74e73a8c50a54dd77451c"
15:58:42.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:42 on dist-test-slave-874w"
15:58:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.633656 456 fs_report.cc:383] FS layout report
15:58:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:58:42.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/wal
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/wal
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/data/data
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:58:42.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:58:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.683813 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:58:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.685477 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:58:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.686014 456 kserver.cc:163] Server-wide thread pool size limit: 3276
15:58:42.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.770637 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:33885
15:58:42.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.770725 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:33885 every 8 connection(s)
15:58:42.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.775058 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/data/info.pb
15:58:42.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.781373 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:58:42.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.782235 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 456
15:58:42.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.782735 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-1/wal/instance
15:58:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.786127 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/wal
15:58:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/data
15:58:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:42.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/logs
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/data/info.pb
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:42.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:41539
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:42.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42183/jwks.json with env {}
15:58:42.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.797856 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } has no permanent_uuid. Determining permanent_uuid...
15:58:42.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.826901 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } has no permanent_uuid. Determining permanent_uuid...
15:58:42.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.844785 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } has no permanent_uuid. Determining permanent_uuid...
15:58:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:42.847637 481 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:42.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:42.851563 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:42.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.901124 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 1
15:58:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:42.904935 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:42.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:42.973417 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 2
15:58:42.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:42.978873 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:43.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.070338 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 3
15:58:43.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.074220 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.134531 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.135128 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:43.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.135726 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:43.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.167367 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:58:43.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.167656 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:43.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.167860 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:58:43.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.168135 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:58:43.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.168332 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:58:43.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.168471 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42183/jwks.json
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.203523 531 master_runner.cc:386] Master server non-default flags:
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/data
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/wal
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:41539
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42183/jwks.json
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/data/info.pb
15:58:43.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/logs
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:43.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:43.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:43.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.204881 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:43.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.207163 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:43.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.217679 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 4
15:58:43.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.221805 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:43.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.231031 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:43.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.294517 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } attempt: 7
15:58:43.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.317593 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } has no permanent_uuid. Determining permanent_uuid...
15:58:43.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.326815 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:43.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.385371 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 1
15:58:43.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.391706 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:43.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.473340 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 2
15:58:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.480464 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.521262 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 5
15:58:43.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.528664 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:43.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.566085 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 3
15:58:43.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.572991 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:43.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:43.713616 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 4
15:58:43.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.719918 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:44.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.003757 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 5
15:58:44.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:44.011068 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.076346 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 6
15:58:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:44.090247 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:44.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.231411 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:44.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:43.238837 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:44.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:44.522310 543 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1289 milliseconds
15:58:44.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.522527 531 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:58:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.523754 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:58:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.526880 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:58:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.528267 531 hybrid_clock.cc:648] HybridClock initialized: now 1790438324528227 us; error 70 us; skew 500 ppm
15:58:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.529162 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:58:44.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.535892 531 webserver.cc:460] Webserver started at http://127.0.93.188:40557/ using document root <none> and password file <none>
15:58:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.536883 531 fs_manager.cc:359] Metadata directory not provided
15:58:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.537158 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:58:44.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.537707 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:58:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.545634 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/data/instance:
15:58:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afb13859f8e8455e8b2d9c19fd411b83"
15:58:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:44 on dist-test-slave-874w"
15:58:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.547389 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/wal/instance:
15:58:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afb13859f8e8455e8b2d9c19fd411b83"
15:58:44.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:44 on dist-test-slave-874w"
15:58:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.557798 531 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.010s sys 0.000s
15:58:44.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.563724 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 6
15:58:44.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.566018 555 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:58:44.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.567472 531 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
15:58:44.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.567821 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/wal
15:58:44.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "afb13859f8e8455e8b2d9c19fd411b83"
15:58:44.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:44 on dist-test-slave-874w"
15:58:44.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.568302 531 fs_report.cc:383] FS layout report
15:58:44.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:58:44.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/wal
15:58:44.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/wal
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/data/data
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:58:44.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:58:44.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:44.571810 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:41539: Network error: Client connection negotiation failed: client connection to 127.0.93.188:41539: connect: Connection refused (error 111)
15:58:44.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.616385 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:58:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.618033 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:58:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.618566 531 kserver.cc:163] Server-wide thread pool size limit: 3276
15:58:44.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.700519 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:41539
15:58:44.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.700598 608 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:41539 every 8 connection(s)
15:58:44.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.703104 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/data/info.pb
15:58:44.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.708962 609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:58:44.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.709463 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 531
15:58:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.709928 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/master-2/wal/instance
15:58:44.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.725543 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } has no permanent_uuid. Determining permanent_uuid...
15:58:44.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.754981 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } has no permanent_uuid. Determining permanent_uuid...
15:58:44.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.767828 609 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } has no permanent_uuid. Determining permanent_uuid...
15:58:44.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.795876 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83: Bootstrap starting.
15:58:44.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.802002 609 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83: Neither blocks nor log segments found. Creating new log.
15:58:44.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.804495 609 log.cc:824] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83: Log is configured to *not* fsync() on all Append() calls
15:58:44.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.809275 609 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83: No bootstrap required, opened a new log
15:58:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.833487 609 raft_consensus.cc:348] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.834149 609 raft_consensus.cc:374] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:58:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.834348 609 raft_consensus.cc:729] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: afb13859f8e8455e8b2d9c19fd411b83, State: Initialized, Role: FOLLOWER
15:58:44.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.835120 609 consensus_queue.cc:260] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:44.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.838207 618 sys_catalog.cc:455] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:44.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.838940 618 sys_catalog.cc:458] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: This master's current role is: FOLLOWER
15:58:44.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.839864 609 sys_catalog.cc:564] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: configured and running, proceeding with master startup.
15:58:44.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:44.867496 629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:58:44.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:44.867794 629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:58:44.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:44.874336 563 tablet.cc:2369] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:58:44.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.864852 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:37821: OK, instance_id { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" instance_seqno: 1790438320919729 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:58:44.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:44.904314 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:58:45.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.150890 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 7
15:58:45.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.175267 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c: Bootstrap starting.
15:58:45.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.183060 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c: Neither blocks nor log segments found. Creating new log.
15:58:45.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.185729 529 log.cc:824] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c: Log is configured to *not* fsync() on all Append() calls
15:58:45.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.190490 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c: No bootstrap required, opened a new log
15:58:45.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.222357 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.223637 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:58:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.223948 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 734e7d30b0e74e73a8c50a54dd77451c, State: Initialized, Role: FOLLOWER
15:58:45.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.225595 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:45.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.230043 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:45.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.231104 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [sys.catalog]: This master's current role is: FOLLOWER
15:58:45.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.231905 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [sys.catalog]: configured and running, proceeding with master startup.
15:58:45.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.277308 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:58:45.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.277988 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:45.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.281229 618 leader_election.cc:288] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers da70e45476d64c0da4573f61137fffdb (127.0.93.190:37821), 734e7d30b0e74e73a8c50a54dd77451c (127.0.93.189:33885)
15:58:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.283401 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "afb13859f8e8455e8b2d9c19fd411b83" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "734e7d30b0e74e73a8c50a54dd77451c" is_pre_election: true
15:58:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.284435 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate afb13859f8e8455e8b2d9c19fd411b83 in term 0.
15:58:45.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:45.285529 644 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:58:45.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:45.286065 644 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:58:45.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.286635 561 leader_election.cc:302] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 734e7d30b0e74e73a8c50a54dd77451c, afb13859f8e8455e8b2d9c19fd411b83; no voters:
15:58:45.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.287655 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:58:45.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.288029 618 raft_consensus.cc:482] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:58:45.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.288380 618 raft_consensus.cc:3037] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 0 FOLLOWER]: Advancing to term 1
15:58:45.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.306304 618 raft_consensus.cc:504] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.256793 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:37821: OK, instance_id { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" instance_seqno: 1790438320919729 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:58:45.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.317649 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:58:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.319957 618 leader_election.cc:288] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [CANDIDATE]: Term 1 election: Requested vote from peers da70e45476d64c0da4573f61137fffdb (127.0.93.190:37821), 734e7d30b0e74e73a8c50a54dd77451c (127.0.93.189:33885)
15:58:45.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.320137 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "afb13859f8e8455e8b2d9c19fd411b83" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "734e7d30b0e74e73a8c50a54dd77451c"
15:58:45.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.320729 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [term 0 FOLLOWER]: Advancing to term 1
15:58:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.329210 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate afb13859f8e8455e8b2d9c19fd411b83 in term 1.
15:58:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.330916 561 leader_election.cc:302] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 734e7d30b0e74e73a8c50a54dd77451c, afb13859f8e8455e8b2d9c19fd411b83; no voters:
15:58:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.331756 618 raft_consensus.cc:2781] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 1 FOLLOWER]: Leader election won for term 1
15:58:45.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.335088 618 raft_consensus.cc:686] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [term 1 LEADER]: Becoming Leader. State: Replica: afb13859f8e8455e8b2d9c19fd411b83, State: Running, Role: LEADER
15:58:45.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.336128 618 consensus_queue.cc:237] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.344225 646 sys_catalog.cc:455] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: SysCatalogTable state changed. Reason: New leader afb13859f8e8455e8b2d9c19fd411b83. Latest consensus state: current_term: 1 leader_uuid: "afb13859f8e8455e8b2d9c19fd411b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.344965 646 sys_catalog.cc:458] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: This master's current role is: LEADER
15:58:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.349901 648 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
15:58:45.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.358682 648 catalog_manager.cc:1484] Initializing Kudu cluster ID...
15:58:45.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.387074 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [term 1 FOLLOWER]: Refusing update from remote peer afb13859f8e8455e8b2d9c19fd411b83: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:58:45.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.388903 618 consensus_queue.cc:1035] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [LEADER]: Connected to new peer: Peer: permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:58:45.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.426375 649 mvcc.cc:205] Tried to move back new op lower bound from 7333635380757635072 to 7333635380587425792. Current Snapshot: MvccSnapshot[applied={T|T < 7333635380757635072}]
15:58:45.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.428354 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [sys.catalog]: SysCatalogTable state changed. Reason: New leader afb13859f8e8455e8b2d9c19fd411b83. Latest consensus state: current_term: 1 leader_uuid: "afb13859f8e8455e8b2d9c19fd411b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:45.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.431622 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [sys.catalog]: This master's current role is: FOLLOWER
15:58:45.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.439366 646 sys_catalog.cc:455] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "afb13859f8e8455e8b2d9c19fd411b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:45.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.440162 646 sys_catalog.cc:458] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: This master's current role is: LEADER
15:58:45.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.442688 648 catalog_manager.cc:1347] Generated new cluster ID: 92555ec021774ad781167df3670d17a3
15:58:45.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.442957 648 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
15:58:45.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.461217 633 sys_catalog.cc:455] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "afb13859f8e8455e8b2d9c19fd411b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:45.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.462100 633 sys_catalog.cc:458] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c [sys.catalog]: This master's current role is: FOLLOWER
15:58:45.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.488864 648 catalog_manager.cc:1370] Generated new certificate authority record
15:58:45.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.492025 648 catalog_manager.cc:1504] Loading token signing keys...
15:58:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.521077 648 catalog_manager.cc:5868] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83: Generated new TSK 0
15:58:45.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.525372 648 catalog_manager.cc:1514] Initializing in-progress tserver states...
15:58:45.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.553306 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/wal
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/data
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/logs
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/data/info.pb
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:45.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:45.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:58:45.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.605240 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } attempt: 7
15:58:45.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.661531 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb: Bootstrap starting.
15:58:45.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.682760 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb: Neither blocks nor log segments found. Creating new log.
15:58:45.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.688896 454 log.cc:824] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb: Log is configured to *not* fsync() on all Append() calls
15:58:45.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.717859 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb: No bootstrap required, opened a new log
15:58:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.783450 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.784325 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:58:45.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.784613 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: da70e45476d64c0da4573f61137fffdb, State: Initialized, Role: FOLLOWER
15:58:45.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.785537 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } }
15:58:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.803434 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.804287 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [sys.catalog]: This master's current role is: FOLLOWER
15:58:45.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.805342 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [sys.catalog]: configured and running, proceeding with master startup.
15:58:45.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.805492 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "afb13859f8e8455e8b2d9c19fd411b83" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da70e45476d64c0da4573f61137fffdb" is_pre_election: true
15:58:45.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.806789 429 raft_consensus.cc:2454] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate afb13859f8e8455e8b2d9c19fd411b83 in term 0.
15:58:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.809092 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "afb13859f8e8455e8b2d9c19fd411b83" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da70e45476d64c0da4573f61137fffdb"
15:58:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.809862 428 raft_consensus.cc:3037] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [term 0 FOLLOWER]: Advancing to term 1
15:58:45.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.820320 428 raft_consensus.cc:2454] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate afb13859f8e8455e8b2d9c19fd411b83 in term 1.
15:58:45.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.822273 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [term 1 FOLLOWER]: Refusing update from remote peer afb13859f8e8455e8b2d9c19fd411b83: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:58:45.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.832218 646 consensus_queue.cc:1035] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [LEADER]: Connected to new peer: Peer: permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:58:45.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:45.874619 408 tablet.cc:2369] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:58:45.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.900406 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [sys.catalog]: SysCatalogTable state changed. Reason: New leader afb13859f8e8455e8b2d9c19fd411b83. Latest consensus state: current_term: 1 leader_uuid: "afb13859f8e8455e8b2d9c19fd411b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:45.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.901171 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [sys.catalog]: This master's current role is: FOLLOWER
15:58:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.901007 665 sys_catalog.cc:455] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "afb13859f8e8455e8b2d9c19fd411b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.903234 665 sys_catalog.cc:458] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb [sys.catalog]: This master's current role is: FOLLOWER
15:58:45.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:45.920001 673 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:58:46.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:46.127848 654 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:46.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:46.129148 654 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:46.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:46.129941 654 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:46.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:46.185948 654 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:46.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:46.187341 654 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
15:58:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.247705 654 tablet_server_runner.cc:78] Tablet server non-default flags:
15:58:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:46.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/data
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/wal
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/data/info.pb
15:58:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
15:58:46.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:46.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/logs
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:46.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:46.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.249346 654 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:46.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.251611 654 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:46.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:46.277781 680 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.294572 644 catalog_manager.cc:1259] Loaded cluster ID: 92555ec021774ad781167df3670d17a3
15:58:46.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.295317 644 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c: loading cluster ID for follower catalog manager: success
15:58:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.305651 644 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c: acquiring CA information for follower catalog manager: success
15:58:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.325909 644 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 734e7d30b0e74e73a8c50a54dd77451c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:58:46.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.422672 646 sys_catalog.cc:455] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "afb13859f8e8455e8b2d9c19fd411b83" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da70e45476d64c0da4573f61137fffdb" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 37821 } } peers { permanent_uuid: "734e7d30b0e74e73a8c50a54dd77451c" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 33885 } } peers { permanent_uuid: "afb13859f8e8455e8b2d9c19fd411b83" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 41539 } } }
15:58:46.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.432533 646 sys_catalog.cc:458] T 00000000000000000000000000000000 P afb13859f8e8455e8b2d9c19fd411b83 [sys.catalog]: This master's current role is: LEADER
15:58:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.938563 673 catalog_manager.cc:1259] Loaded cluster ID: 92555ec021774ad781167df3670d17a3
15:58:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.938817 673 catalog_manager.cc:1552] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb: loading cluster ID for follower catalog manager: success
15:58:46.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.950601 673 catalog_manager.cc:1574] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb: acquiring CA information for follower catalog manager: success
15:58:46.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:46.956096 673 catalog_manager.cc:1602] T 00000000000000000000000000000000 P da70e45476d64c0da4573f61137fffdb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:58:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:47.672349 678 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 654
15:58:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:48.142819 681 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1863 milliseconds
15:58:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:48.143575 654 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.867s user 0.000s sys 0.004s
15:58:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:48.144909 654 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.868s user 0.000s sys 0.004s
15:58:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:46.277812 679 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:48.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.149384 654 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:58:48.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:48.150671 683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:48.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.152293 654 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:58:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.163134 654 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:58:48.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.167382 654 hybrid_clock.cc:648] HybridClock initialized: now 1790438328167341 us; error 64 us; skew 500 ppm
15:58:48.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.168712 654 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:58:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.199251 654 webserver.cc:460] Webserver started at http://127.0.93.129:37035/ using document root <none> and password file <none>
15:58:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.200855 654 fs_manager.cc:359] Metadata directory not provided
15:58:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.201359 654 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:58:48.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.206545 654 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:58:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.213842 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/data/instance:
15:58:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1cabcb002faa48a0ab74b3f4488d7c95"
15:58:48.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:48 on dist-test-slave-874w"
15:58:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.215862 654 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/wal/instance:
15:58:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1cabcb002faa48a0ab74b3f4488d7c95"
15:58:48.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:48 on dist-test-slave-874w"
15:58:48.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.240113 654 fs_manager.cc:691] Time spent creating directory manager: real 0.023s user 0.009s sys 0.008s
15:58:48.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.259902 694 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:58:48.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.261505 654 fs_manager.cc:725] Time spent opening block manager: real 0.016s user 0.004s sys 0.008s
15:58:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.261897 654 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/wal
15:58:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1cabcb002faa48a0ab74b3f4488d7c95"
15:58:48.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:48 on dist-test-slave-874w"
15:58:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.262907 654 fs_report.cc:383] FS layout report
15:58:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:58:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/wal
15:58:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/wal
15:58:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/data/data
15:58:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:58:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:58:48.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:58:48.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:58:48.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.364892 654 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:58:48.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.367435 654 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:58:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.368081 654 kserver.cc:163] Server-wide thread pool size limit: 3276
15:58:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.377315 654 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:58:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.395795 654 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:58:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.396096 654 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:58:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.396381 654 ts_tablet_manager.cc:610] Registered 0 tablets
15:58:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.396595 654 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:58:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.820863 654 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:39881
15:58:48.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.821009 806 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:39881 every 8 connection(s)
15:58:48.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.824409 654 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/data/info.pb
15:58:48.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.833225 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 654
15:58:48.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.834153 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-0/wal/instance
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.856815 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/wal
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/data
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/logs
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/data/info.pb
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:48.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:48.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:58:48.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.926116 808 heartbeater.cc:347] Connected to a master server at 127.0.93.188:41539
15:58:48.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.926637 808 heartbeater.cc:464] Registering TS with master...
15:58:48.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.927772 808 heartbeater.cc:511] Master 127.0.93.188:41539 requested a full tablet report, sending...
15:58:48.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.929991 807 heartbeater.cc:347] Connected to a master server at 127.0.93.189:33885
15:58:48.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.930052 811 heartbeater.cc:347] Connected to a master server at 127.0.93.190:37821
15:58:48.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.930400 807 heartbeater.cc:464] Registering TS with master...
15:58:48.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.930487 811 heartbeater.cc:464] Registering TS with master...
15:58:48.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.931321 807 heartbeater.cc:511] Master 127.0.93.189:33885 requested a full tablet report, sending...
15:58:48.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.931509 811 heartbeater.cc:511] Master 127.0.93.190:37821 requested a full tablet report, sending...
15:58:48.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.934326 574 ts_manager.cc:194] Registered new tserver with Master: 1cabcb002faa48a0ab74b3f4488d7c95 (127.0.93.129:39881)
15:58:48.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.934633 494 ts_manager.cc:194] Registered new tserver with Master: 1cabcb002faa48a0ab74b3f4488d7c95 (127.0.93.129:39881)
15:58:48.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.934921 419 ts_manager.cc:194] Registered new tserver with Master: 1cabcb002faa48a0ab74b3f4488d7c95 (127.0.93.129:39881)
15:58:48.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:48.937294 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:45965
15:58:49.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:49.258973 815 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:49.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:49.259438 815 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:49.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:49.259869 815 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:49.292279 815 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:49.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:49.293082 815 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
15:58:49.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:49.331532 815 tablet_server_runner.cc:78] Tablet server non-default flags:
15:58:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:49.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/data
15:58:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/wal
15:58:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
15:58:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:49.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/data/info.pb
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/logs
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:49.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:49.332830 815 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:49.334649 815 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:49.360380 827 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:49.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:49.942230 808 heartbeater.cc:503] Master 127.0.93.188:41539 was elected leader, sending a full tablet report...
15:58:50.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:49.367769 825 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:50.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:49.360926 824 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:50.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:50.777176 826 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1415 milliseconds
15:58:50.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.777288 815 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:58:50.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.778618 815 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:58:50.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.780782 815 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:58:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.782186 815 hybrid_clock.cc:648] HybridClock initialized: now 1790438330782138 us; error 60 us; skew 500 ppm
15:58:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.783038 815 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:58:50.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.790335 815 webserver.cc:460] Webserver started at http://127.0.93.130:39031/ using document root <none> and password file <none>
15:58:50.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.791602 815 fs_manager.cc:359] Metadata directory not provided
15:58:50.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.791872 815 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:58:50.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.792382 815 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:58:50.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.797326 815 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/data/instance:
15:58:50.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc5b5fef2e448a98d2d6eaba07cecf4"
15:58:50.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:50 on dist-test-slave-874w"
15:58:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.798734 815 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/wal/instance:
15:58:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc5b5fef2e448a98d2d6eaba07cecf4"
15:58:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:50 on dist-test-slave-874w"
15:58:50.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.805730 815 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.002s sys 0.005s
15:58:50.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.812004 835 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:58:50.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.813186 815 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.813555 815 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/wal
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2dc5b5fef2e448a98d2d6eaba07cecf4"
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:50 on dist-test-slave-874w"
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.813952 815 fs_report.cc:383] FS layout report
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/wal
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/wal
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/data/data
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:58:50.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:58:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:58:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:58:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:58:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:58:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:58:50.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:58:50.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.860595 815 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:58:50.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.862718 815 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:58:50.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.863265 815 kserver.cc:163] Server-wide thread pool size limit: 3276
15:58:50.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.866000 815 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:58:50.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.870594 815 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:58:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.870843 815 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:58:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.871326 815 ts_tablet_manager.cc:610] Registered 0 tablets
15:58:50.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:50.871542 815 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:58:51.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.042770 815 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:37625
15:58:51.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.042867 947 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:37625 every 8 connection(s)
15:58:51.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.045460 815 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/data/info.pb
15:58:51.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.055174 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 815
15:58:51.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.056030 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-1/wal/instance
15:58:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.073173 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/wal
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/data
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/logs
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/data/info.pb
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:51.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:51.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:51.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:51.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:51.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:58:51.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.099998 949 heartbeater.cc:347] Connected to a master server at 127.0.93.188:41539
15:58:51.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.100509 949 heartbeater.cc:464] Registering TS with master...
15:58:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.101776 949 heartbeater.cc:511] Master 127.0.93.188:41539 requested a full tablet report, sending...
15:58:51.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.103955 950 heartbeater.cc:347] Connected to a master server at 127.0.93.190:37821
15:58:51.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.104295 950 heartbeater.cc:464] Registering TS with master...
15:58:51.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.105103 950 heartbeater.cc:511] Master 127.0.93.190:37821 requested a full tablet report, sending...
15:58:51.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.106153 574 ts_manager.cc:194] Registered new tserver with Master: 2dc5b5fef2e448a98d2d6eaba07cecf4 (127.0.93.130:37625)
15:58:51.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.107784 948 heartbeater.cc:347] Connected to a master server at 127.0.93.189:33885
15:58:51.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.107784 419 ts_manager.cc:194] Registered new tserver with Master: 2dc5b5fef2e448a98d2d6eaba07cecf4 (127.0.93.130:37625)
15:58:51.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.108103 948 heartbeater.cc:464] Registering TS with master...
15:58:51.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.108640 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:34121
15:58:51.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.109300 948 heartbeater.cc:511] Master 127.0.93.189:33885 requested a full tablet report, sending...
15:58:51.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.111941 494 ts_manager.cc:194] Registered new tserver with Master: 2dc5b5fef2e448a98d2d6eaba07cecf4 (127.0.93.130:37625)
15:58:51.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:51.403802 958 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:51.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:51.404264 958 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:51.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:51.404698 958 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:51.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:51.435755 958 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:51.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:51.436550 958 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.472659 958 tablet_server_runner.cc:78] Tablet server non-default flags:
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:43421
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/data
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/wal
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/data/info.pb
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:37821,127.0.93.189:33885,127.0.93.188:41539
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/logs
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:51.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:51.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.474010 958 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:51.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.476115 958 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:51.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:51.495242 965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:52.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.112692 949 heartbeater.cc:503] Master 127.0.93.188:41539 was elected leader, sending a full tablet report...
15:58:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:51.495281 966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:51.495324 968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:52.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:51.497582 958 server_base.cc:1034] running on GCE node
15:58:52.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.767747 958 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:58:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.773298 958 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:58:52.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.774770 958 hybrid_clock.cc:648] HybridClock initialized: now 1790438332774718 us; error 51 us; skew 500 ppm
15:58:52.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.775806 958 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:58:52.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.783509 958 webserver.cc:460] Webserver started at http://127.0.93.131:38789/ using document root <none> and password file <none>
15:58:52.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.784665 958 fs_manager.cc:359] Metadata directory not provided
15:58:52.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.784917 958 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:58:52.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.785427 958 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:58:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.792779 958 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/data/instance:
15:58:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bc7ef48231c849ccb7c86391b2238a04"
15:58:52.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:52 on dist-test-slave-874w"
15:58:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.794211 958 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/wal/instance:
15:58:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bc7ef48231c849ccb7c86391b2238a04"
15:58:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:52 on dist-test-slave-874w"
15:58:52.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.803113 958 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
15:58:52.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.809955 975 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:58:52.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.811194 958 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
15:58:52.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.811493 958 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/wal
15:58:52.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bc7ef48231c849ccb7c86391b2238a04"
15:58:52.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:52 on dist-test-slave-874w"
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.811828 958 fs_report.cc:383] FS layout report
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/wal
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/wal
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/data/data
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:58:52.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:58:52.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.849550 958 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:58:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.850977 958 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:58:52.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.851382 958 kserver.cc:163] Server-wide thread pool size limit: 3276
15:58:52.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.853726 958 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:58:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.858107 958 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:58:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.858295 958 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:58:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.858510 958 ts_tablet_manager.cc:610] Registered 0 tablets
15:58:52.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:52.858651 958 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:58:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.027112 958 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:33441
15:58:53.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.027202 1087 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:33441 every 8 connection(s)
15:58:53.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.029475 958 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/data/info.pb
15:58:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.034327 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 958
15:58:53.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.034811 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster9094069032798001370/ts-2/wal/instance
15:58:53.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.087775 1088 heartbeater.cc:347] Connected to a master server at 127.0.93.189:33885
15:58:53.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.088431 1088 heartbeater.cc:464] Registering TS with master...
15:58:53.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.090057 1088 heartbeater.cc:511] Master 127.0.93.189:33885 requested a full tablet report, sending...
15:58:53.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.094357 494 ts_manager.cc:194] Registered new tserver with Master: bc7ef48231c849ccb7c86391b2238a04 (127.0.93.131:33441)
15:58:53.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.095194 1089 heartbeater.cc:347] Connected to a master server at 127.0.93.188:41539
15:58:53.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.095530 1089 heartbeater.cc:464] Registering TS with master...
15:58:53.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.096342 1089 heartbeater.cc:511] Master 127.0.93.188:41539 requested a full tablet report, sending...
15:58:53.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.098480 574 ts_manager.cc:194] Registered new tserver with Master: bc7ef48231c849ccb7c86391b2238a04 (127.0.93.131:33441)
15:58:53.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.100468 574 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:45457
15:58:53.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.101238 1092 heartbeater.cc:347] Connected to a master server at 127.0.93.190:37821
15:58:53.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.101568 1092 heartbeater.cc:464] Registering TS with master...
15:58:53.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.102380 1092 heartbeater.cc:511] Master 127.0.93.190:37821 requested a full tablet report, sending...
15:58:53.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.104528 419 ts_manager.cc:194] Registered new tserver with Master: bc7ef48231c849ccb7c86391b2238a04 (127.0.93.131:33441)
15:58:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.109030 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
15:58:53.125 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:58:53.127 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:58:53.141 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 37821
}
bound_http_address {
host: "127.0.93.190"
port: 34913
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 33885
}
bound_http_address {
host: "127.0.93.189"
port: 44231
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 41539
}
bound_http_address {
host: "127.0.93.188"
port: 40557
}
}
}
15:58:53.146 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:58:53.149 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 39881
}
bound_http_address {
host: "127.0.93.129"
port: 37035
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 37625
}
bound_http_address {
host: "127.0.93.130"
port: 39031
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 33441
}
bound_http_address {
host: "127.0.93.131"
port: 38789
}
}
}
15:58:53.149 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:58:53.324 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:58:53.441 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskDtRWV2/test-tmp
15:58:53.442 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:58:53.442 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskDtRWV2/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:58:53.447 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:58:53.800 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:58:53.800 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:58:53.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.803449 1100 mini_oidc.cc:152] Starting JWKS server
15:58:53.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.807219 1100 webserver.cc:460] Webserver started at http://127.0.0.1:36535/ using document root <none> and password file <none>
15:58:53.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.807672 1100 mini_oidc.cc:189] Starting OIDC Discovery server
15:58:53.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:53.809603 1100 webserver.cc:460] Webserver started at http://127.0.0.1:38645/ using document root <none> and password file <none>
15:58:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:58:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/principal' for realm 'KRBTEST.COM',
15:58:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:58:53.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:53 dist-test-slave-874w krb5kdc[1112](info): setting up network...
15:58:53.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:58:53.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:53 dist-test-slave-874w krb5kdc[1112](info): set up 2 sockets
15:58:53.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:53 dist-test-slave-874w krb5kdc[1112](info): commencing operation
15:58:53.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:58:54.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:54.105190 1089 heartbeater.cc:503] Master 127.0.93.188:41539 was elected leader, sending a full tablet report...
15:58:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:54.929675 1100 mini_kdc.cc:121] Time spent starting KDC: real 1.095s user 0.007s sys 0.015s
15:58:54.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:58:54.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:58:54.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:58:54.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:58:54.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:58:54.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:58:55.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:58:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:58:55.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:58:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:58:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/test-user.keytab.
15:58:55.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/test-user.keytab.
15:58:55.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:55 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438335, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:58:55.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:58:55.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T15:58:55Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:58:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T15:58:55Z Disabled control of system clock
15:58:55.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.62@KRBTEST.COM; defaulting to no policy
15:58:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:58:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.62@KRBTEST.COM" created.
15:58:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:58:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.62.keytab.
15:58:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.62.keytab.
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:55.135635 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/logs
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/info.pb
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:38541
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.62.keytab
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:58:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:58:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:58:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:58:55.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:58:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.465534 1125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.466069 1125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:55.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.466657 1125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.497017 1125 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:58:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.497555 1125 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:58:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.497860 1125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.498056 1125 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:58:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.498324 1125 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:58:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.498581 1125 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:58:55.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.498740 1125 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36535/jwks.json
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:55.536669 1125 master_runner.cc:386] Master server non-default flags:
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.62.keytab
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:38541
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/info.pb
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/logs
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:55.537774 1125 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:55.538750 1125 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:55.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.558594 1132 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:56.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.558679 1134 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:56.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:55.560684 1125 server_base.cc:1034] running on GCE node
15:58:56.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:55.558656 1131 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:56.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.785077 1125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:58:56.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.788316 1125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:58:56.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.789951 1125 hybrid_clock.cc:648] HybridClock initialized: now 1790438336789875 us; error 76 us; skew 500 ppm
15:58:56.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.791338 1125 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:58:56.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:56 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438336, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:58:56.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.799363 1125 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
15:58:56.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.805008 1125 webserver.cc:460] Webserver started at http://127.1.19.62:41235/ using document root <none> and password file <none>
15:58:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.805975 1125 fs_manager.cc:359] Metadata directory not provided
15:58:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.806188 1125 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:58:56.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.806802 1125 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:58:56.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.811681 1125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/instance:
15:58:56.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6acd83c36ce43c7b9f3788e8de1dc80"
15:58:56.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:56 on dist-test-slave-874w"
15:58:56.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.812940 1125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal/instance:
15:58:56.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6acd83c36ce43c7b9f3788e8de1dc80"
15:58:56.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:56 on dist-test-slave-874w"
15:58:56.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.821246 1125 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.002s sys 0.007s
15:58:56.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.828125 1144 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:58:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.829371 1125 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
15:58:56.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.829666 1125 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6acd83c36ce43c7b9f3788e8de1dc80"
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:56 on dist-test-slave-874w"
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.829964 1125 fs_report.cc:383] FS layout report
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/data
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:58:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:58:56.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.871035 1125 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:58:56.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.876624 1125 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:58:56.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.877022 1125 kserver.cc:163] Server-wide thread pool size limit: 3276
15:58:56.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.957649 1125 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:38541
15:58:56.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.957796 1195 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:38541 every 8 connection(s)
15:58:56.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.960574 1125 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/info.pb
15:58:56.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.967355 1196 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:58:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.969996 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 1125
15:58:56.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.970551 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal/instance
15:58:56.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:56.980712 1196 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } has no permanent_uuid. Determining permanent_uuid...
15:58:56.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.61@KRBTEST.COM; defaulting to no policy
15:58:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:58:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.61@KRBTEST.COM" created.
15:58:57.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:57 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438336, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
15:58:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:58:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.61.keytab.
15:58:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.61.keytab.
15:58:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.019480 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/logs
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/info.pb
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:45403
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.61.keytab
15:58:57.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:58:57.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.021179 1196 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } has no permanent_uuid. Determining permanent_uuid...
15:58:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.023350 1148 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:58:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.027842 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45403: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:58:57.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.077315 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } attempt: 1
15:58:57.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.081736 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45403: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:58:57.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.150136 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } attempt: 2
15:58:57.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.154685 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45403: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:58:57.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.246147 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } attempt: 3
15:58:57.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.250372 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45403: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:58:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.354548 1203 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:57.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.355376 1203 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:57.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.356042 1203 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:57.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.387470 1203 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:58:57.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.387712 1203 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:58:57.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.387969 1203 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:57.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.388161 1203 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:58:57.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.388347 1203 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:58:57.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.388520 1203 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:58:57.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.388796 1203 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36535/jwks.json
15:58:57.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.393834 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } attempt: 4
15:58:57.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.397389 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45403: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.427286 1203 master_runner.cc:386] Master server non-default flags:
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:58:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.61.keytab
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:45403
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/info.pb
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:57.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/logs
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:57.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.428431 1203 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:57.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.429828 1203 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:57.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.452476 1212 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:57.696848 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } attempt: 5
15:58:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.708125 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45403: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:58:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.255745 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } attempt: 6
15:58:58.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:58.266629 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45403: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:58:58.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.452857 1209 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:58.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:57.452677 1210 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:58.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:58.868552 1211 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1415 milliseconds
15:58:58.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.868662 1203 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:58:58.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.870056 1203 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:58:58.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.873589 1203 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:58:58.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.875017 1203 hybrid_clock.cc:648] HybridClock initialized: now 1790438338874963 us; error 85 us; skew 500 ppm
15:58:58.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.875816 1203 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:58:58.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:58 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438338, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:58:58.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.888762 1203 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
15:58:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.894681 1203 webserver.cc:460] Webserver started at http://127.1.19.61:33659/ using document root <none> and password file <none>
15:58:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.895884 1203 fs_manager.cc:359] Metadata directory not provided
15:58:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.896080 1203 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:58:58.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.896484 1203 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:58:58.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.901253 1203 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/instance:
15:58:58.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a615e858443546d8bb1d56810e620431"
15:58:58.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:58 on dist-test-slave-874w"
15:58:58.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.902563 1203 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal/instance:
15:58:58.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a615e858443546d8bb1d56810e620431"
15:58:58.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:58 on dist-test-slave-874w"
15:58:58.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.910326 1203 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
15:58:58.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.916544 1222 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:58:58.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.917629 1203 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
15:58:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.918013 1203 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:58:58.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a615e858443546d8bb1d56810e620431"
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:58 on dist-test-slave-874w"
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.918385 1203 fs_report.cc:383] FS layout report
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/data
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:58:58.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:58:58.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.985719 1203 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:58:58.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.994007 1203 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:58:58.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:58.994717 1203 kserver.cc:163] Server-wide thread pool size limit: 3276
15:58:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.091537 1203 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:45403
15:58:59.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.091686 1274 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:45403 every 8 connection(s)
15:58:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.094704 1203 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/info.pb
15:58:59.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.096469 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 1203
15:58:59.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.097437 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal/instance
15:58:59.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.101428 1275 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:58:59.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.60@KRBTEST.COM; defaulting to no policy
15:58:59.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.114343 1275 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } has no permanent_uuid. Determining permanent_uuid...
15:58:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:58:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.60@KRBTEST.COM" created.
15:58:59.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:59 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438338, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
15:58:59.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.188092 1275 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } has no permanent_uuid. Determining permanent_uuid...
15:58:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:59 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438338, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
15:58:59.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.260907 1275 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } has no permanent_uuid. Determining permanent_uuid...
15:58:59.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.263056 1225 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.266372 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:58:59.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.60.keytab.
15:58:59.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.60.keytab.
15:58:59.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.287027 1100 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.1.19.60: real 0.187s user 0.000s sys 0.011s
15:58:59.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.288550 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/logs
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/info.pb
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:38793
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.60.keytab
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:58:59.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:58:59.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.315896 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 1
15:58:59.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.319680 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.327214 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } attempt: 7
15:58:59.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:58:59 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438336, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
15:58:59.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.351257 1196 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } has no permanent_uuid. Determining permanent_uuid...
15:58:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.356932 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.388182 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 2
15:58:59.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.392891 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.415333 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 1
15:58:59.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.418952 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.484330 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 3
15:58:59.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.488801 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.500349 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 2
15:58:59.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.504243 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.589629 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 3
15:58:59.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.593902 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.619270 1284 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:58:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.619828 1284 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:58:59.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.620534 1284 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:58:59.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.632215 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 4
15:58:59.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.636838 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.656450 1284 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:58:59.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.656672 1284 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:58:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.656941 1284 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:58:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.657140 1284 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:58:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.657331 1284 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:58:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.657517 1284 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:58:59.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.657686 1284 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36535/jwks.json
15:58:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.695355 1284 master_runner.cc:386] Master server non-default flags:
15:58:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:58:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:58:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:58:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.60.keytab
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:38793
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/info.pb
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/logs
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:58:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.696521 1284 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:58:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.697508 1284 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:58:59.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.731441 1293 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:58:59.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.734341 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 4
15:58:59.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.738811 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:58:59.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:58:59.936350 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 5
15:58:59.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.941367 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:59:00.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:00.022257 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 5
15:59:00.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:00.027349 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:59:00.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:00.488857 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 6
15:59:00.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:00.497901 1275 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:59:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:00.579946 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 6
15:59:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:00.587489 1196 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:38793: Network error: Client connection negotiation failed: client connection to 127.1.19.60:38793: connect: Connection refused (error 111)
15:59:01.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.732854 1294 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:01.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:58:59.731467 1296 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:01.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:01.055397 1295 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:59:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.055554 1284 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:59:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.060029 1284 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:59:01.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.063030 1284 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:59:01.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.064486 1284 hybrid_clock.cc:648] HybridClock initialized: now 1790438341064428 us; error 55 us; skew 500 ppm
15:59:01.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.065456 1284 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:59:01.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438341, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.072713 1284 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
15:59:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.078871 1284 webserver.cc:460] Webserver started at http://127.1.19.60:34807/ using document root <none> and password file <none>
15:59:01.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.079885 1284 fs_manager.cc:359] Metadata directory not provided
15:59:01.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.080206 1284 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:59:01.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.080677 1284 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:59:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.086102 1284 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/instance:
15:59:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5568e2acf7445d8a74e34a1d3ed6907"
15:59:01.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:59:01 on dist-test-slave-874w"
15:59:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.087642 1284 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal/instance:
15:59:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5568e2acf7445d8a74e34a1d3ed6907"
15:59:01.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:59:01 on dist-test-slave-874w"
15:59:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.099370 1284 fs_manager.cc:691] Time spent creating directory manager: real 0.011s user 0.003s sys 0.009s
15:59:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.107784 1309 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:59:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.109019 1284 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.109642 1284 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5568e2acf7445d8a74e34a1d3ed6907"
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:59:01 on dist-test-slave-874w"
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.110093 1284 fs_report.cc:383] FS layout report
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/data
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:59:01.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:59:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:59:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:59:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:59:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:59:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:59:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:59:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:59:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:59:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.169672 1284 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:59:01.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.176415 1284 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:59:01.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.177045 1284 kserver.cc:163] Server-wide thread pool size limit: 3276
15:59:01.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.266155 1284 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:38793
15:59:01.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.266235 1361 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:38793 every 8 connection(s)
15:59:01.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.270277 1284 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/info.pb
15:59:01.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.273710 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 1284
15:59:01.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.274344 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal/instance
15:59:01.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.277894 1362 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:59:01.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.306079 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } has no permanent_uuid. Determining permanent_uuid...
15:59:01.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438335, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
15:59:01.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438341, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
15:59:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.342283 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } has no permanent_uuid. Determining permanent_uuid...
15:59:01.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438335, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
15:59:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438341, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
15:59:01.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.372505 1362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } has no permanent_uuid. Determining permanent_uuid...
15:59:01.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438341, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
15:59:01.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438335, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
15:59:01.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.408924 1362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: Bootstrap starting.
15:59:01.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.410203 1100 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
15:59:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.415777 1362 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: Neither blocks nor log segments found. Creating new log.
15:59:01.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.417908 1362 log.cc:824] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: Log is configured to *not* fsync() on all Append() calls
15:59:01.419 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:59:01.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.424049 1362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: No bootstrap required, opened a new log
15:59:01.434 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:59:01.436 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:59:01.440 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.62"
port: 38541
}
bound_http_address {
host: "127.1.19.62"
port: 41235
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.61"
port: 45403
}
bound_http_address {
host: "127.1.19.61"
port: 33659
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.60"
port: 38793
}
bound_http_address {
host: "127.1.19.60"
port: 34807
}
}
}
15:59:01.441 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:59:01.441 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:01.441 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:59:01.444 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:59:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.454152 1362 raft_consensus.cc:348] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.454988 1362 raft_consensus.cc:374] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:59:01.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.455201 1362 raft_consensus.cc:729] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5568e2acf7445d8a74e34a1d3ed6907, State: Initialized, Role: FOLLOWER
15:59:01.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.456032 1362 consensus_queue.cc:260] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.459398 1374 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } } }
15:59:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.460382 1374 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [sys.catalog]: This master's current role is: FOLLOWER
15:59:01.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.461331 1362 sys_catalog.cc:564] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [sys.catalog]: configured and running, proceeding with master startup.
15:59:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438341, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:59:01.483 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:01.485 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc
15:59:01.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:01.494194 1386 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:01.494 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:59:01.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:01.494601 1386 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:01.496 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc
15:59:01.502 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:59:01.507 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:59:01.509 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:59:01.509 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:59:01.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.488687 1284 master_runner.cc:186] Error getting master registration for 127.1.19.62:38541: OK, instance_id { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" instance_seqno: 1790438336932957 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:59:01.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.518735 1284 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:59:01.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:01 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438341, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:01.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:59:01.556 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:01.556 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc
15:59:01.559 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:59:01.559 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.61:45403
15:59:01.561 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:59:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.558295 1275 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 7
15:59:01.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.562481 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 1203
15:59:01.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:01.591502 1365 negotiation.cc:337] Failed RPC negotiation. Trace:
15:59:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 15:59:01.561198 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:57216 (local address 127.1.19.60:38793)
15:59:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 15:59:01.562326 (+ 1128us) server_negotiation.cc:206] Beginning negotiation
15:59:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 15:59:01.562352 (+ 26us) server_negotiation.cc:399] Waiting for connection header
15:59:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0926 15:59:01.590009 (+ 27657us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57216: BlockingRecv error: recv got EOF from 127.0.0.1:57216 (error 108)
15:59:01.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":158}
15:59:01.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:01.593 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.60:38793
15:59:01.593 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:59:01.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.594362 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 1284
15:59:01.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.621086 1196 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } attempt: 7
15:59:01.625 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:01.626 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.62:38541
15:59:01.626 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:59:01.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.626770 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 1125
15:59:01.655 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:01.655 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.61:45403
15:59:01.657 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:59:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:01.659157 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:59:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:59:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:59:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data
15:59:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:59:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:59:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:59:01.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/logs
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/info.pb
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:45403
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33659
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.61.keytab
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
15:59:01.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:59:01.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:59:01.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:01.991511 1392 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:59:01.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:01.992118 1392 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:59:01.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:01.992713 1392 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:59:02.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.025441 1392 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:59:02.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.025744 1392 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:59:02.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.026162 1392 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:59:02.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.026569 1392 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:59:02.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.026858 1392 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:59:02.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.027125 1392 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:59:02.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.027274 1392 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36535/jwks.json
15:59:02.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:02.065409 1392 master_runner.cc:386] Master server non-default flags:
15:59:02.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:59:02.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.61.keytab
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:45403
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/info.pb
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33659
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/logs
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:59:02.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:59:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:59:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:59:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:02.066622 1392 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:59:02.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:02.067569 1392 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:59:02.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.086087 1399 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:03.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.086822 1398 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:02.088598 1392 server_base.cc:1034] running on GCE node
15:59:03.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:02.086977 1401 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:03.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.305043 1392 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:59:03.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.307912 1392 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:59:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.309303 1392 hybrid_clock.cc:648] HybridClock initialized: now 1790438343309262 us; error 51 us; skew 500 ppm
15:59:03.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.310160 1392 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:59:03.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:03 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438343, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.316653 1392 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
15:59:03.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.322688 1392 webserver.cc:460] Webserver started at http://127.1.19.61:33659/ using document root <none> and password file <none>
15:59:03.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.323634 1392 fs_manager.cc:359] Metadata directory not provided
15:59:03.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.323845 1392 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:59:03.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.332645 1392 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.002s sys 0.003s
15:59:03.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.337148 1410 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:59:03.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.338581 1392 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.338922 1392 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a615e858443546d8bb1d56810e620431"
15:59:03.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:58 on dist-test-slave-874w"
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.341830 1392 fs_report.cc:383] FS layout report
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/data
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:59:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:59:03.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.395790 1392 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:59:03.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.401387 1392 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:59:03.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.401830 1392 kserver.cc:163] Server-wide thread pool size limit: 3276
15:59:03.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.474399 1392 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:45403
15:59:03.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.474927 1461 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:45403 every 8 connection(s)
15:59:03.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.477335 1392 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/data/info.pb
15:59:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.487349 1462 sys_catalog.cc:263] Verifying existing consensus state
15:59:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.487484 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 1392
15:59:03.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:03.488 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.60:38793
15:59:03.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:59:03.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 15:59:03.488185 1462 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:59:03.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.488618 1392 master.cc:537] Master@127.1.19.61:45403 shutting down...
15:59:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.490011 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:59:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:59:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/logs
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/info.pb
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:38793
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34807
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.60.keytab
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:59:03.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:59:03.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:59:03.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.549525 1392 master.cc:552] Master@127.1.19.61:45403 shutdown complete.
15:59:03.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:59:03.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.845840 1464 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:59:03.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.846376 1464 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:59:03.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.847048 1464 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:59:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.879837 1464 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:59:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.880069 1464 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:59:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.880340 1464 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:59:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.880537 1464 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:59:03.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.880715 1464 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:59:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.880909 1464 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:59:03.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.881052 1464 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36535/jwks.json
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.919044 1464 master_runner.cc:386] Master server non-default flags:
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:59:03.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.60.keytab
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:38793
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:59:03.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/info.pb
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34807
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/logs
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.920203 1464 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:59:03.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.921317 1464 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:59:03.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.939882 1474 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:05.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:03.941406 1464 server_base.cc:1034] running on GCE node
15:59:05.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.939908 1472 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:05.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:03.940769 1471 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.240331 1464 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:59:05.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.243244 1464 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:59:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.244712 1464 hybrid_clock.cc:648] HybridClock initialized: now 1790438345244660 us; error 60 us; skew 500 ppm
15:59:05.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.245633 1464 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:59:05.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:05 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438345, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:05.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.259075 1464 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
15:59:05.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.264963 1464 webserver.cc:460] Webserver started at http://127.1.19.60:34807/ using document root <none> and password file <none>
15:59:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.265949 1464 fs_manager.cc:359] Metadata directory not provided
15:59:05.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.266207 1464 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:59:05.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.274138 1464 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
15:59:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.278674 1482 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:59:05.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.279924 1464 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
15:59:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.280311 1464 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:59:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e5568e2acf7445d8a74e34a1d3ed6907"
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:59:01 on dist-test-slave-874w"
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.282836 1464 fs_report.cc:383] FS layout report
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/wal
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/data
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:59:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:59:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:59:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:59:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:59:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:59:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:59:05.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:59:05.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.325917 1464 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:59:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.332113 1464 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:59:05.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.332607 1464 kserver.cc:163] Server-wide thread pool size limit: 3276
15:59:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.420641 1464 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:38793
15:59:05.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.420823 1533 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:38793 every 8 connection(s)
15:59:05.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.425756 1464 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-2/data/info.pb
15:59:05.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.433378 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 1464
15:59:05.434 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:05.434 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.62:38541
15:59:05.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:59:05.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.436579 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/logs
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/info.pb
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:38541
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41235
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.62.keytab
15:59:05.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:59:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:59:05.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:59:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.440444 1534 sys_catalog.cc:263] Verifying existing consensus state
15:59:05.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.447464 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: Bootstrap starting.
15:59:05.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.479585 1534 log.cc:824] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: Log is configured to *not* fsync() on all Append() calls
15:59:05.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.484239 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:59:05.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.484977 1534 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: Bootstrap complete.
15:59:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.509258 1534 raft_consensus.cc:348] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:05.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.511636 1534 raft_consensus.cc:374] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:59:05.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.511919 1534 raft_consensus.cc:729] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5568e2acf7445d8a74e34a1d3ed6907, State: Initialized, Role: FOLLOWER
15:59:05.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.512760 1534 consensus_queue.cc:260] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:05.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.515730 1538 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } } }
15:59:05.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.516563 1538 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [sys.catalog]: This master's current role is: FOLLOWER
15:59:05.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.517390 1534 sys_catalog.cc:564] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [sys.catalog]: configured and running, proceeding with master startup.
15:59:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.547142 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.547837 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.551030 1464 master_runner.cc:186] Error getting master registration for 127.1.19.62:38541: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111),
15:59:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.580941 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:59:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.588128 1488 tablet.cc:2369] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:59:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.804800 1536 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:59:05.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.805565 1536 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:59:05.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.806784 1536 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:59:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.845789 1536 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:59:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.846014 1536 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:59:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.846271 1536 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:59:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.846560 1536 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:59:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.846765 1536 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:59:05.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.846949 1536 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:59:05.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.847084 1536 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36535/jwks.json
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.884681 1536 master_runner.cc:386] Master server non-default flags:
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:35725
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:38541,127.1.19.61:45403,127.1.19.60:38793
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/krb5kdc/kudu_127.1.19.62.keytab
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:38541
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:59:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36535/jwks.json
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/info.pb
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41235
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/logs
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:59:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 26 Sep 2026 14:44:58 UTC on ccycloud.quasar-hxpbms.root.comops.site
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4674114
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.885785 1536 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:59:05.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.886919 1536 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:59:05.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.907155 1558 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:05.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.978391 1538 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:59:05.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.979008 1538 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:05.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.984077 1483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:05.989853 1538 leader_election.cc:288] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541), a615e858443546d8bb1d56810e620431 (127.1.19.61:45403)
15:59:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:06.060470 1486 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a615e858443546d8bb1d56810e620431 (127.1.19.61:45403): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:59:06.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:06.063052 1483 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.1.19.62:38541, user_credentials={real_user=slave}} blocked reactor thread for 79160.1us
15:59:06.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:06.067807 1483 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541): Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:06.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:06.082686 1483 leader_election.cc:302] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5568e2acf7445d8a74e34a1d3ed6907; no voters: a615e858443546d8bb1d56810e620431, d6acd83c36ce43c7b9f3788e8de1dc80
15:59:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:06.084285 1538 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:59:06.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:06.553975 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:06.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:06.555171 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:07.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:07.028985 1530 debug-util.cc:398] Leaking SignalData structure 0x7b0800073060 after lost signal to thread 1466
15:59:07.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:07.029681 1530 debug-util.cc:398] Leaking SignalData structure 0x7b08000737c0 after lost signal to thread 1533
15:59:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.910480 1557 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:07.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:07.425596 1559 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1520 milliseconds
15:59:07.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.425637 1536 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:59:07.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:05.907889 1560 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:59:07.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.431041 1536 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:59:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.434230 1536 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:59:07.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.435889 1536 hybrid_clock.cc:648] HybridClock initialized: now 1790438347435822 us; error 86 us; skew 500 ppm
15:59:07.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.436784 1536 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:59:07.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:07 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438347, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:07.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.443760 1536 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
15:59:07.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.456842 1536 webserver.cc:460] Webserver started at http://127.1.19.62:41235/ using document root <none> and password file <none>
15:59:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.457841 1536 fs_manager.cc:359] Metadata directory not provided
15:59:07.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.458096 1536 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:59:07.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.468686 1536 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.008s sys 0.000s
15:59:07.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.474259 1570 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:59:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.475545 1536 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
15:59:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.475899 1536 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data,/tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:59:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d6acd83c36ce43c7b9f3788e8de1dc80"
15:59:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-09-26 15:58:56 on dist-test-slave-874w"
15:59:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.478251 1536 fs_report.cc:383] FS layout report
15:59:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:59:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/data
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:59:07.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:59:07.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.533130 1536 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:59:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.539732 1536 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:59:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.540251 1536 kserver.cc:163] Server-wide thread pool size limit: 3276
15:59:07.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:07.559247 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:07.559878 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.651829 1536 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:38541
15:59:07.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.651988 1621 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:38541 every 8 connection(s)
15:59:07.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.656160 1536 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/data/info.pb
15:59:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.659438 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu as pid 1536
15:59:07.660 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:59:07.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.703606 1622 sys_catalog.cc:263] Verifying existing consensus state
15:59:07.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260926 15:59:07.704602 1622 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:59:07.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.705111 1536 master.cc:537] Master@127.1.19.62:38541 shutting down...
15:59:07.793 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.831713 1536 master.cc:552] Master@127.1.19.62:38541 shutdown complete.
15:59:07.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskDtRWV2/test-tmp/mini-kudu-cluster4832040306258764571/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:59:07.897 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.60:38793(127.1.19.60:38793)] connecting to peer
15:59:07.899 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:07.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.956898 1627 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:59:07.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.959750 1627 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:07.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:07.968113 1627 leader_election.cc:288] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541), a615e858443546d8bb1d56810e620431 (127.1.19.61:45403)
15:59:07.979 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.1.19.60:38793(127.1.19.60:38793)] Successfully connected to peer
15:59:07.995 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:07.995 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:08.010082 1486 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a615e858443546d8bb1d56810e620431 (127.1.19.61:45403): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:59:08.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:08.015352 1483 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541): Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:08.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:08.015745 1483 leader_election.cc:302] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5568e2acf7445d8a74e34a1d3ed6907; no voters: a615e858443546d8bb1d56810e620431, d6acd83c36ce43c7b9f3788e8de1dc80
15:59:08.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:08.016433 1627 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:59:08.024 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.026 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.026 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.027 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.164 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:59:08.174 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.60
15:59:08.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:08 dist-test-slave-874w krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438341, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
15:59:08.514 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xe818e553, L:/127.0.0.1:44508 - R:/127.1.19.60:38793] using SASL/GSSAPI
15:59:08.535 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:08.554 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:08.559 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:08.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:08.564754 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:08.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:08.565107 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:08.567 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.568 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.568 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.577 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.578 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.579 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.580 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:08.593 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:08.594 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:08.595 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.596 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.597 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.602 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.603 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.607 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.607 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:08.613 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:08.614 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:08.616 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.616 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.617 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.624 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.624 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.625 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.625 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:08.653 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:08.654 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:08.658 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.658 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.658 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.660 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.660 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.661 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.664 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:08.693 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:08.695 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:08.697 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.697 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.697 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.698 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.699 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.699 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.701 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:08.752 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:08.755 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:08.757 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.758 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.758 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.760 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.760 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.761 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.762 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:08.813 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:08.815 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:08.818 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.818 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.819 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:08.820 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.820 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.821 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:08.821 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:09.053 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:09.057 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:09.059 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.059 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.060 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.060 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.060 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.061 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.063 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:09.093 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:09.095 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:09.096 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.096 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.097 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.098 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.099 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.100 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.102 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:09.452 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:09.454 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:09.456 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.457 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.457 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:09.458 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.458 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.458 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:09.460 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:09.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:09.569960 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:09.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:09.570353 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:09.691776 1635 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:59:09.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:09.692143 1635 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:09.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:09.693580 1635 leader_election.cc:288] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541), a615e858443546d8bb1d56810e620431 (127.1.19.61:45403)
15:59:09.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:09.699402 1483 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541): Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:09.699846 1486 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a615e858443546d8bb1d56810e620431 (127.1.19.61:45403): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:59:09.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:09.700165 1486 leader_election.cc:302] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5568e2acf7445d8a74e34a1d3ed6907; no voters: a615e858443546d8bb1d56810e620431, d6acd83c36ce43c7b9f3788e8de1dc80
15:59:09.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:09.700680 1635 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:59:10.352 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:10.354 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:10.356 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:10.356 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:10.357 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:10.357 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:10.357 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:10.357 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:10.359 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:10.573364 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:10.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:10.573614 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:10.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:10.590459 1464 master_runner.cc:186] Error getting master registration for 127.1.19.62:38541: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111),
15:59:10.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:10.590869 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:59:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:11.576596 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:11.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:11.576875 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:12.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:12.579828 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:12.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:12.580089 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:12.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:12.805964 1645 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:59:12.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:12.806499 1645 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:12.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:12.808000 1645 leader_election.cc:288] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541), a615e858443546d8bb1d56810e620431 (127.1.19.61:45403)
15:59:12.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:12.810144 1486 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111) [suppressed 5 similar messages]
15:59:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:12.812847 1486 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a615e858443546d8bb1d56810e620431 (127.1.19.61:45403): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:59:12.813 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:12.813472 1483 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541): Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:12.813884 1483 leader_election.cc:302] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5568e2acf7445d8a74e34a1d3ed6907; no voters: a615e858443546d8bb1d56810e620431, d6acd83c36ce43c7b9f3788e8de1dc80
15:59:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:12.814599 1645 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:59:12.814 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:12.816 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:12.816 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:12.816 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:12.817 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:12.817 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:12.818 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:12.818 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:13.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:13.583552 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:13.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:13.583815 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:14.373 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:14.374 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:14.376 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:14.376 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:14.376 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:14.376 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:14.376 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:14.376 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:14.379 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:14.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:14.587183 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:14.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:14.587571 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:14.893 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:14.894 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:14.896 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:14.896 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:14.896 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:14.897 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:14.897 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:14.897 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:14.898 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:15.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:15 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438355, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:15.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:15.262928 1478 init.cc:295] Successfully reacquired a new kerberos TGT
15:59:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:15.590833 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:15.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:15.591140 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:15.594018 1464 master_runner.cc:186] Error getting master registration for 127.1.19.62:38541: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111),
15:59:15.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:15.594411 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:59:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:16.594354 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:16.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:16.594662 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:17.352 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:17.354 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:17.355 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:17.355 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:17.356 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:17.357 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:17.357 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:17.357 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:17.358 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:17.597834 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:17.598063 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:18.600889 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:18.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:18.601217 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:19.604067 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:19.604326 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:20.125595 1658 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:59:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:20.126005 1658 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:20.129271 1658 leader_election.cc:288] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541), a615e858443546d8bb1d56810e620431 (127.1.19.61:45403)
15:59:20.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:20.130336 1483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:59:20.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:20.134223 1483 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541): Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:20.134958 1486 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a615e858443546d8bb1d56810e620431 (127.1.19.61:45403): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:59:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:20.135511 1486 leader_election.cc:302] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5568e2acf7445d8a74e34a1d3ed6907; no voters: a615e858443546d8bb1d56810e620431, d6acd83c36ce43c7b9f3788e8de1dc80
15:59:20.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:20.136397 1658 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:59:20.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:20.407567 1084 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4ba0 after lost signal to thread 960
15:59:20.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:20.408730 1084 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4f80 after lost signal to thread 1087
15:59:20.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:20.596798 1464 master_runner.cc:186] Error getting master registration for 127.1.19.62:38541: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111),
15:59:20.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:20.597252 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:59:20.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:20.607527 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:20.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:20.607774 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:21.212 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:21.215 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:21.215 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.215 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.216 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.218 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:21.218 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:21.218 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:21.219 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:21.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:21.610720 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:21.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:21.610976 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:21.732 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:21.737 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:21.736 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.737 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.738 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.739 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:21.739 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:21.739 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:21.745 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:21.993 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:21.995 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:21.996 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.996 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.996 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:21.998 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:21.998 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:21.998 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:22.000 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:22.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:22.613619 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:22.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:22.613864 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:23.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:23.616576 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:23.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:23.616839 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:23.772 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:23.774 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:23.776 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:23.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:23.777 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:23.777 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:23.777 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:23.777 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:23.778 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:23.912 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:23.913 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:23.914 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:23.914 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:23.914 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:23.914 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:23.915 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:23.915 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:23.917 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:24.620455 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:24.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:24.620862 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:25.232 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:25.234 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:25.235 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:25.235 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:25.235 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:25.235 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:25.235 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:25.235 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:25.238 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:25 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438365, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:25.265319 1478 init.cc:295] Successfully reacquired a new kerberos TGT
15:59:25.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:25.536098 1670 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:59:25.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:25.536597 1670 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:25.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:25.538268 1670 leader_election.cc:288] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541), a615e858443546d8bb1d56810e620431 (127.1.19.61:45403)
15:59:25.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:25.541191 1483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:59:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:25.545390 1483 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541): Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:25.549196 1486 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a615e858443546d8bb1d56810e620431 (127.1.19.61:45403): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:59:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:25.549635 1486 leader_election.cc:302] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5568e2acf7445d8a74e34a1d3ed6907; no voters: a615e858443546d8bb1d56810e620431, d6acd83c36ce43c7b9f3788e8de1dc80
15:59:25.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:25.550338 1670 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:59:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:25.623692 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:25.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:25.623988 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:25.635839 1464 master_runner.cc:186] Error getting master registration for 127.1.19.62:38541: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111),
15:59:25.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:25.636250 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:59:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:26.046712 1530 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3200 after lost signal to thread 1466
15:59:26.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:26.047348 1530 debug-util.cc:398] Leaking SignalData structure 0x7b080009b2a0 after lost signal to thread 1533
15:59:26.433 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:26.434 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:26.435 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:26.435 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:26.435 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:26.435 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:26.435 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:26.435 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:26.437 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:26.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:26.626693 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:26.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:26.626940 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:27.512018 1675 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:59:27.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:27.512501 1675 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:27.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:27.514727 1675 leader_election.cc:288] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541), a615e858443546d8bb1d56810e620431 (127.1.19.61:45403)
15:59:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:27.519630 1483 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541): Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:27.520104 1486 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a615e858443546d8bb1d56810e620431 (127.1.19.61:45403): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:59:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:27.520459 1486 leader_election.cc:302] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5568e2acf7445d8a74e34a1d3ed6907; no voters: a615e858443546d8bb1d56810e620431, d6acd83c36ce43c7b9f3788e8de1dc80
15:59:27.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:27.521147 1675 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:59:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:27.630023 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:27.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:27.630313 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:28.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:28.633471 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:28.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:28.633798 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:29.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:29.637619 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:29.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:29.638273 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:30.472 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:30.473 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:30.474 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:30.474 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:30.474 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:30.474 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:30.474 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:30.474 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:30.477 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:30.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:30.638962 1464 master_runner.cc:186] Error getting master registration for 127.1.19.62:38541: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111),
15:59:30.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:30.639415 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:59:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:30.641731 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:30.642071 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:31.645398 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:31.645674 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:32.072 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:32.073 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:32.074 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:32.074 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:32.074 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:32.074 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:32.075 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:32.075 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:32.076 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:32.648500 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:32.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:32.648756 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:33.652065 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:33.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:33.652331 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:34.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:34.656176 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:34.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:34.656533 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 26 15:59:35 dist-test-slave-874w krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1790438375, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:59:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:35.267619 1478 init.cc:295] Successfully reacquired a new kerberos TGT
15:59:35.472 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:35.473 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:35.474 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:35.474 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:35.474 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:35.474 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:35.475 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:35.475 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:35.477 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:35.643103 1464 master_runner.cc:186] Error getting master registration for 127.1.19.62:38541: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111),
15:59:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:35.643692 1464 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:59:35.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:35.659029 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:35.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:35.659278 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:35.761664 1689 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:59:35.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:35.762127 1689 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d6acd83c36ce43c7b9f3788e8de1dc80" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 38541 } } peers { permanent_uuid: "a615e858443546d8bb1d56810e620431" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45403 } } peers { permanent_uuid: "e5568e2acf7445d8a74e34a1d3ed6907" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 38793 } }
15:59:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:35.763885 1689 leader_election.cc:288] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541), a615e858443546d8bb1d56810e620431 (127.1.19.61:45403)
15:59:35.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:35.765138 1483 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111) [suppressed 3 similar messages]
15:59:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:35.768105 1483 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer d6acd83c36ce43c7b9f3788e8de1dc80 (127.1.19.62:38541): Network error: Client connection negotiation failed: client connection to 127.1.19.62:38541: connect: Connection refused (error 111)
15:59:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:35.768707 1486 leader_election.cc:334] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer a615e858443546d8bb1d56810e620431 (127.1.19.61:45403): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45403: connect: Connection refused (error 111)
15:59:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:35.769043 1486 leader_election.cc:302] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e5568e2acf7445d8a74e34a1d3ed6907; no voters: a615e858443546d8bb1d56810e620431, d6acd83c36ce43c7b9f3788e8de1dc80
15:59:35.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:35.769883 1689 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:59:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:36.661625 1549 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e5568e2acf7445d8a74e34a1d3ed6907: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:59:36.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260926 15:59:36.661876 1549 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:59:37.232 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45403(127.1.19.61:45403)] connecting to peer
15:59:37.233 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:38541(127.1.19.62:38541)] connecting to peer
15:59:37.234 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:37.234 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:37.234 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45403(127.1.19.61:45403)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:37.235 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:38541(127.1.19.62:38541)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:37.235 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45403: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403
15:59:37.235 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:38541: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
15:59:37.237 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541; will retry
15:59:37.241 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29573), Traces: [0ms] refreshing cache from master, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [319ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [349ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [853ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [858ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [891ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [899ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [903ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [915ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [919ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [925ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [926ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [930ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [935ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [940ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [946ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [947ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [948ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [981ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [986ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [988ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1075ms] refreshing cache from master, [1075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1084ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1085ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1135ms] refreshing cache from master, [1136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1145ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1375ms] refreshing cache from master, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1382ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1385ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1386ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1415ms] refreshing cache from master, [1415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1425ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1775ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1781ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1782ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1783ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [2675ms] refreshing cache from master, [2675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [2676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [2677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [2680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [2680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [2682ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [2682ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [5135ms] refreshing cache from master, [5136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [5136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [5137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [5139ms] too many traces: truncated at 100 traces, deferred=Deferred@1798979020(state=PENDING, result=null, callback=(continuation of Deferred@86070081 after retry RPC after error@1391222986) -> (continuation of Deferred@45484096 after retry RPC after error@953717127) -> (continuation of Deferred@929855110 after retry RPC after error@112700049) -> (continuation of Deferred@879842831 after retry RPC after error@1222225327) -> (continuation of Deferred@475343476 after retry RPC after error@655226957) -> (continuation of Deferred@426906509 after retry RPC after error@482566617) -> (continuation of Deferred@703659711 after retry RPC after error@1635234908) -> (continuation of Deferred@1754283396 after retry RPC after error@1876575462) -> (continuation of Deferred@1106332324 after retry RPC after error@494453650) -> (continuation of Deferred@1953341860 after retry RPC after error@1227695803) -> (continuation of Deferred@37941584 after retry RPC after error@1509803737) -> (continuation of Deferred@1082408131 after retry RPC after error@1614594710) -> (continuation of Deferred@1977710512 after retry RPC after error@1109245147) -> (continuation of Deferred@597619327 after retry RPC after error@623951080) -> (continuation of Deferred@1527911177 after retry RPC after error@1950779348) -> (continuation of Deferred@538943017 after retry RPC after error@6857022) -> (continuation of Deferred@1261006945 after retry RPC after error@194066167) -> (continuation of Deferred@1461915514 after retry RPC after error@302462449) -> (continuation of Deferred@304997800 after retry RPC after error@277438356) -> (continuation of Deferred@1604246048 after retry RPC after error@1948107935) -> (continuation of Deferred@165703441 after retry RPC after error@1406137414) -> (continuation of Deferred@2049539013 after retry RPC after error@1619685667) -> (continuation of Deferred@1788351172 after retry RPC after error@1180284437) -> (continuation of Deferred@2006911560 after retry RPC after error@1143963377) -> (continuation of Deferred@1854622338 after retry RPC after error@1701421144) -> (continuation of Deferred@1789606903 after retry RPC after error@1431046309), errback=(continuation of Deferred@86070081 after retry RPC after error@1391222986) -> (continuation of Deferred@45484096 after retry RPC after error@953717127) -> (continuation of Deferred@929855110 after retry RPC after error@112700049) -> (continuation of Deferred@879842831 after retry RPC after error@1222225327) -> (continuation of Deferred@475343476 after retry RPC after error@655226957) -> (continuation of Deferred@426906509 after retry RPC after error@482566617) -> (continuation of Deferred@703659711 after retry RPC after error@1635234908) -> (continuation of Deferred@1754283396 after retry RPC after error@1876575462) -> (continuation of Deferred@1106332324 after retry RPC after error@494453650) -> (continuation of Deferred@1953341860 after retry RPC after error@1227695803) -> (continuation of Deferred@37941584 after retry RPC after error@1509803737) -> (continuation of Deferred@1082408131 after retry RPC after error@1614594710) -> (continuation of Deferred@1977710512 after retry RPC after error@1109245147) -> (continuation of Deferred@597619327 after retry RPC after error@623951080) -> (continuation of Deferred@1527911177 after retry RPC after error@1950779348) -> (continuation of Deferred@538943017 after retry RPC after error@6857022) -> (continuation of Deferred@1261006945 after retry RPC after error@194066167) -> (continuation of Deferred@1461915514 after retry RPC after error@302462449) -> (continuation of Deferred@304997800 after retry RPC after error@277438356) -> (continuation of Deferred@1604246048 after retry RPC after error@1948107935) -> (continuation of Deferred@165703441 after retry RPC after error@1406137414) -> (continuation of Deferred@2049539013 after retry RPC after error@1619685667) -> (continuation of Deferred@1788351172 after retry RPC after error@1180284437) -> (continuation of Deferred@2006911560 after retry RPC after error@1143963377) -> (continuation of Deferred@1854622338 after retry RPC after error@1701421144) -> (continuation of Deferred@1789606903 after retry RPC after error@1431046309)))
15:59:37.246 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.60:38793(127.1.19.60:38793)] disconnecting while in state READY
15:59:37.247 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.61:45403(127.1.19.61:45403)] disconnecting while in state TERMINATED
15:59:37.247 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.62:38541(127.1.19.62:38541)] disconnecting while in state TERMINATED
15:59:37.252 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:59:37.254 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:59:37.254 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.1.19.60:38793(127.1.19.60:38793)] handling channelInactive
15:59:37.255 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.60:38793(127.1.19.60:38793)] cleaning up while in state READY due to: connection closed
15:59:37.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:37.264707 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 1464
15:59:38.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T15:59:38Z chronyd exiting
15:59:39.472 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:59:39.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:39.474360 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 654
15:59:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:39.499816 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 815
15:59:39.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:39.526458 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 958
15:59:39.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:39.555214 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 386
15:59:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:39.585350 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 456
15:59:39.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260926 15:59:39.615430 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskDtRWV2/build/tsan/bin/kudu with pid 531
15:59:40.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-09-26T15:59:40Z chronyd exiting
15:59:41.830 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29573), Traces: [0ms] refreshing cache from master, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [319ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [349ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [853ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [858ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [891ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [899ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [903ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [915ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [919ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [925ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [926ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [930ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [935ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [940ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [946ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [947ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [948ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [981ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [986ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [988ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1075ms] refreshing cache from master, [1075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1084ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1085ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1135ms] refreshing cache from master, [1136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1145ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1375ms] refreshing cache from master, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1382ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1385ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1386ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1415ms] refreshing cache from master, [1415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1425ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1775ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1781ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1782ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1783ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [2675ms] refreshing cache from master, [2675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [2676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [2677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [2680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [2680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [2682ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [2682ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [5135ms] refreshing cache from master, [5136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [5136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [5137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [5139ms] too many traces: truncated at 100 traces, deferred=Deferred@1798979020(state=PENDING, result=null, callback=(continuation of Deferred@86070081 after retry RPC after error@1391222986) -> (continuation of Deferred@45484096 after retry RPC after error@953717127) -> (continuation of Deferred@929855110 after retry RPC after error@112700049) -> (continuation of Deferred@879842831 after retry RPC after error@1222225327) -> (continuation of Deferred@475343476 after retry RPC after error@655226957) -> (continuation of Deferred@426906509 after retry RPC after error@482566617) -> (continuation of Deferred@703659711 after retry RPC after error@1635234908) -> (continuation of Deferred@1754283396 after retry RPC after error@1876575462) -> (continuation of Deferred@1106332324 after retry RPC after error@494453650) -> (continuation of Deferred@1953341860 after retry RPC after error@1227695803) -> (continuation of Deferred@37941584 after retry RPC after error@1509803737) -> (continuation of Deferred@1082408131 after retry RPC after error@1614594710) -> (continuation of Deferred@1977710512 after retry RPC after error@1109245147) -> (continuation of Deferred@597619327 after retry RPC after error@623951080) -> (continuation of Deferred@1527911177 after retry RPC after error@1950779348) -> (continuation of Deferred@538943017 after retry RPC after error@6857022) -> (continuation of Deferred@1261006945 after retry RPC after error@194066167) -> (continuation of Deferred@1461915514 after retry RPC after error@302462449) -> (continuation of Deferred@304997800 after retry RPC after error@277438356) -> (continuation of Deferred@1604246048 after retry RPC after error@1948107935) -> (continuation of Deferred@165703441 after retry RPC after error@1406137414) -> (continuation of Deferred@2049539013 after retry RPC after error@1619685667) -> (continuation of Deferred@1788351172 after retry RPC after error@1180284437) -> (continuation of Deferred@2006911560 after retry RPC after error@1143963377) -> (continuation of Deferred@1854622338 after retry RPC after error@1701421144) -> (continuation of Deferred@1789606903 after retry RPC after error@1431046309), errback=(continuation of Deferred@86070081 after retry RPC after error@1391222986) -> (continuation of Deferred@45484096 after retry RPC after error@953717127) -> (continuation of Deferred@929855110 after retry RPC after error@112700049) -> (continuation of Deferred@879842831 after retry RPC after error@1222225327) -> (continuation of Deferred@475343476 after retry RPC after error@655226957) -> (continuation of Deferred@426906509 after retry RPC after error@482566617) -> (continuation of Deferred@703659711 after retry RPC after error@1635234908) -> (continuation of Deferred@1754283396 after retry RPC after error@1876575462) -> (continuation of Deferred@1106332324 after retry RPC after error@494453650) -> (continuation of Deferred@1953341860 after retry RPC after error@1227695803) -> (continuation of Deferred@37941584 after retry RPC after error@1509803737) -> (continuation of Deferred@1082408131 after retry RPC after error@1614594710) -> (continuation of Deferred@1977710512 after retry RPC after error@1109245147) -> (continuation of Deferred@597619327 after retry RPC after error@623951080) -> (continuation of Deferred@1527911177 after retry RPC after error@1950779348) -> (continuation of Deferred@538943017 after retry RPC after error@6857022) -> (continuation of Deferred@1261006945 after retry RPC after error@194066167) -> (continuation of Deferred@1461915514 after retry RPC after error@302462449) -> (continuation of Deferred@304997800 after retry RPC after error@277438356) -> (continuation of Deferred@1604246048 after retry RPC after error@1948107935) -> (continuation of Deferred@165703441 after retry RPC after error@1406137414) -> (continuation of Deferred@2049539013 after retry RPC after error@1619685667) -> (continuation of Deferred@1788351172 after retry RPC after error@1180284437) -> (continuation of Deferred@2006911560 after retry RPC after error@1143963377) -> (continuation of Deferred@1854622338 after retry RPC after error@1701421144) -> (continuation of Deferred@1789606903 after retry RPC after error@1431046309)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.62:38541
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29573), Traces: [0ms] refreshing cache from master, [91ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [222ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [319ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [349ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [853ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [858ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [876ms] refreshing cache from master, [877ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [879ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [882ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [891ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [899ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [901ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [903ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [915ms] refreshing cache from master, [915ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [919ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [925ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [926ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [930ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [935ms] refreshing cache from master, [935ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [937ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [940ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [946ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [947ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [948ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [975ms] refreshing cache from master, [975ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [981ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [983ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [986ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [988ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1015ms] refreshing cache from master, [1016ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1017ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1020ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1022ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1023ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1024ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1075ms] refreshing cache from master, [1075ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1077ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1078ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1081ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1084ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1085ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1135ms] refreshing cache from master, [1136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1141ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1143ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1145ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1375ms] refreshing cache from master, [1376ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1377ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1380ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1382ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1383ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1385ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1386ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1415ms] refreshing cache from master, [1415ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1416ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1417ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1420ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1422ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1424ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1425ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1775ms] refreshing cache from master, [1775ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [1776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [1777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [1780ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [1781ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [1782ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [1783ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [2675ms] refreshing cache from master, [2675ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [2676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [2677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [2680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:38541: Network error: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541, [2680ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45403: Network error: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [2682ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:38793: OK, [2682ms] delaying RPC due to: Service unavailable: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403, [5135ms] refreshing cache from master, [5136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45403, [5136ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:38793, [5137ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:38541, [5139ms] too many traces: truncated at 100 traces, deferred=Deferred@1798979020(state=PENDING, result=null, callback=(continuation of Deferred@86070081 after retry RPC after error@1391222986) -> (continuation of Deferred@45484096 after retry RPC after error@953717127) -> (continuation of Deferred@929855110 after retry RPC after error@112700049) -> (continuation of Deferred@879842831 after retry RPC after error@1222225327) -> (continuation of Deferred@475343476 after retry RPC after error@655226957) -> (continuation of Deferred@426906509 after retry RPC after error@482566617) -> (continuation of Deferred@703659711 after retry RPC after error@1635234908) -> (continuation of Deferred@1754283396 after retry RPC after error@1876575462) -> (continuation of Deferred@1106332324 after retry RPC after error@494453650) -> (continuation of Deferred@1953341860 after retry RPC after error@1227695803) -> (continuation of Deferred@37941584 after retry RPC after error@1509803737) -> (continuation of Deferred@1082408131 after retry RPC after error@1614594710) -> (continuation of Deferred@1977710512 after retry RPC after error@1109245147) -> (continuation of Deferred@597619327 after retry RPC after error@623951080) -> (continuation of Deferred@1527911177 after retry RPC after error@1950779348) -> (continuation of Deferred@538943017 after retry RPC after error@6857022) -> (continuation of Deferred@1261006945 after retry RPC after error@194066167) -> (continuation of Deferred@1461915514 after retry RPC after error@302462449) -> (continuation of Deferred@304997800 after retry RPC after error@277438356) -> (continuation of Deferred@1604246048 after retry RPC after error@1948107935) -> (continuation of Deferred@165703441 after retry RPC after error@1406137414) -> (continuation of Deferred@2049539013 after retry RPC after error@1619685667) -> (continuation of Deferred@1788351172 after retry RPC after error@1180284437) -> (continuation of Deferred@2006911560 after retry RPC after error@1143963377) -> (continuation of Deferred@1854622338 after retry RPC after error@1701421144) -> (continuation of Deferred@1789606903 after retry RPC after error@1431046309), errback=(continuation of Deferred@86070081 after retry RPC after error@1391222986) -> (continuation of Deferred@45484096 after retry RPC after error@953717127) -> (continuation of Deferred@929855110 after retry RPC after error@112700049) -> (continuation of Deferred@879842831 after retry RPC after error@1222225327) -> (continuation of Deferred@475343476 after retry RPC after error@655226957) -> (continuation of Deferred@426906509 after retry RPC after error@482566617) -> (continuation of Deferred@703659711 after retry RPC after error@1635234908) -> (continuation of Deferred@1754283396 after retry RPC after error@1876575462) -> (continuation of Deferred@1106332324 after retry RPC after error@494453650) -> (continuation of Deferred@1953341860 after retry RPC after error@1227695803) -> (continuation of Deferred@37941584 after retry RPC after error@1509803737) -> (continuation of Deferred@1082408131 after retry RPC after error@1614594710) -> (continuation of Deferred@1977710512 after retry RPC after error@1109245147) -> (continuation of Deferred@597619327 after retry RPC after error@623951080) -> (continuation of Deferred@1527911177 after retry RPC after error@1950779348) -> (continuation of Deferred@538943017 after retry RPC after error@6857022) -> (continuation of Deferred@1261006945 after retry RPC after error@194066167) -> (continuation of Deferred@1461915514 after retry RPC after error@302462449) -> (continuation of Deferred@304997800 after retry RPC after error@277438356) -> (continuation of Deferred@1604246048 after retry RPC after error@1948107935) -> (continuation of Deferred@165703441 after retry RPC after error@1406137414) -> (continuation of Deferred@2049539013 after retry RPC after error@1619685667) -> (continuation of Deferred@1788351172 after retry RPC after error@1180284437) -> (continuation of Deferred@2006911560 after retry RPC after error@1143963377) -> (continuation of Deferred@1854622338 after retry RPC after error@1701421144) -> (continuation of Deferred@1789606903 after retry RPC after error@1431046309)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.61:45403,127.1.19.60:38793,127.1.19.62:38541) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45403(127.1.19.61:45403): Connection refused: /127.1.19.61:45403,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:38541(127.1.19.62:38541): Connection refused: /127.1.19.62:38541
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.62:38541
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)