15:27:17.576 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:27:17.588 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNRqU_n/test-tmp
15:27:17.590 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:27:17.594 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNRqU_n/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:27:17.808 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:27:17.985 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:27:17.986 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:27:17.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:17.991350 12597 mini_oidc.cc:152] Starting JWKS server
15:27:17.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:17.995617 12597 webserver.cc:460] Webserver started at http://127.0.0.1:37303/ using document root <none> and password file <none>
15:27:17.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:17.996084 12597 mini_oidc.cc:189] Starting OIDC Discovery server
15:27:17.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:17.997753 12597 webserver.cc:460] Webserver started at http://127.0.0.1:38195/ using document root <none> and password file <none>
15:27:18.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:27:18Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:27:18.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:27:18Z Disabled control of system clock
15:27:18.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:18.045363 12597 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:18.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:18.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/wal
15:27:18.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/data
15:27:18.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:18.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:18.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:18.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/logs
15:27:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/data/info.pb
15:27:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:18.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:18.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.126:35005
15:27:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.126
15:27:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:18.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37303/jwks.json with env {}
15:27:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.335985 12609 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.336568 12609 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.337216 12609 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.367172 12609 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:18.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.367432 12609 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.367700 12609 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.367951 12609 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.368233 12609 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:18.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.368453 12609 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37303/jwks.json
15:27:18.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:18.402127 12609 master_runner.cc:386] Master server non-default flags:
15:27:18.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:18.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:18.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:18.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:18.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/data
15:27:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/wal
15:27:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:18.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.126:35005
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37303/jwks.json
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:18.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/data/info.pb
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.126
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/logs
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:18.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:18.403453 12609 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:18.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:18.405495 12609 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:18.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.423841 12618 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.423834 12616 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:18.425539 12615 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:19.625219 12617 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1198 milliseconds
15:27:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.625332 12609 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:27:19.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.626703 12609 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:19.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.629297 12609 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:19.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.630726 12609 hybrid_clock.cc:648] HybridClock initialized: now 1781969239630680 us; error 50 us; skew 500 ppm
15:27:19.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.631618 12609 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.641121 12609 webserver.cc:460] Webserver started at http://127.12.77.126:40845/ using document root <none> and password file <none>
15:27:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.642095 12609 fs_manager.cc:359] Metadata directory not provided
15:27:19.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.642333 12609 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:19.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.642866 12609 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:19.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.647688 12609 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/data/instance:
15:27:19.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f433500da0bf4d1295647e387a7bf1df"
15:27:19.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:19 on dist-test-slave-fpc8"
15:27:19.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.648947 12609 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/wal/instance:
15:27:19.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f433500da0bf4d1295647e387a7bf1df"
15:27:19.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:19 on dist-test-slave-fpc8"
15:27:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.655952 12609 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.008s sys 0.000s
15:27:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.661274 12625 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.662376 12609 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
15:27:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.662715 12609 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/wal
15:27:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f433500da0bf4d1295647e387a7bf1df"
15:27:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:19 on dist-test-slave-fpc8"
15:27:19.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.663141 12609 fs_report.cc:383] FS layout report
15:27:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/wal
15:27:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/wal
15:27:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/data/data
15:27:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:19.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:19.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:19.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.716640 12609 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.718214 12609 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:19.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.718719 12609 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:19.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.804607 12609 rpc_server.cc:307] RPC server started. Bound to: 127.12.77.126:35005
15:27:19.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.804701 12676 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.77.126:35005 every 8 connection(s)
15:27:19.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.807473 12609 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/data/info.pb
15:27:19.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.812976 12677 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:27:19.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.813127 12597 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 12609
15:27:19.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.813612 12597 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-0/wal/instance
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.818857 12597 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/wal
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/data
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:19.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/logs
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/data/info.pb
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:19.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.125:44219
15:27:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.125
15:27:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:19.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:19.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:19.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:19.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:19.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:19.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37303/jwks.json with env {}
15:27:19.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.828493 12677 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } has no permanent_uuid. Determining permanent_uuid...
15:27:19.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.856163 12677 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } has no permanent_uuid. Determining permanent_uuid...
15:27:19.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:19.858424 12626 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.77.125:44219: connect: Connection refused (error 111)
15:27:19.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:19.862583 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.125:44219: Network error: Client connection negotiation failed: client connection to 127.12.77.125:44219: connect: Connection refused (error 111)
15:27:19.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.912164 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } attempt: 1
15:27:19.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:19.916203 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.125:44219: Network error: Client connection negotiation failed: client connection to 127.12.77.125:44219: connect: Connection refused (error 111)
15:27:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:19.984679 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } attempt: 2
15:27:19.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:19.989370 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.125:44219: Network error: Client connection negotiation failed: client connection to 127.12.77.125:44219: connect: Connection refused (error 111)
15:27:20.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:20.080858 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } attempt: 3
15:27:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.085425 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.125:44219: Network error: Client connection negotiation failed: client connection to 127.12.77.125:44219: connect: Connection refused (error 111)
15:27:20.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.127262 12679 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.127810 12679 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.128432 12679 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:20.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.158239 12679 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:20.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.158501 12679 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:20.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.158690 12679 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.158865 12679 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.159152 12679 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:20.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.159305 12679 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37303/jwks.json
15:27:20.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:20.193296 12679 master_runner.cc:386] Master server non-default flags:
15:27:20.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/data
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/wal
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.125:44219
15:27:20.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37303/jwks.json
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/data/info.pb
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.125
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:20.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:20.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:20.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:20.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/logs
15:27:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:20.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:20.194537 12679 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:20.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:20.196199 12679 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.212282 12689 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:20.228789 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } attempt: 4
15:27:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.232275 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.125:44219: Network error: Client connection negotiation failed: client connection to 127.12.77.125:44219: connect: Connection refused (error 111)
15:27:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:20.531922 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } attempt: 5
15:27:20.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.536041 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.125:44219: Network error: Client connection negotiation failed: client connection to 127.12.77.125:44219: connect: Connection refused (error 111)
15:27:21.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.083628 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } attempt: 6
15:27:21.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.094070 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.125:44219: Network error: Client connection negotiation failed: client connection to 127.12.77.125:44219: connect: Connection refused (error 111)
15:27:21.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.212500 12688 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:20.212653 12691 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:20.214032 12679 server_base.cc:1034] running on GCE node
15:27:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.401597 12679 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:21.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.404917 12679 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:21.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.406347 12679 hybrid_clock.cc:648] HybridClock initialized: now 1781969241406315 us; error 63 us; skew 500 ppm
15:27:21.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.407217 12679 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:21.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.413580 12679 webserver.cc:460] Webserver started at http://127.12.77.125:39967/ using document root <none> and password file <none>
15:27:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.414436 12679 fs_manager.cc:359] Metadata directory not provided
15:27:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.414613 12679 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:21.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.415082 12679 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.419956 12679 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/data/instance:
15:27:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "367a150421b34b3493e473c47f1dcef9"
15:27:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:21 on dist-test-slave-fpc8"
15:27:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.421052 12679 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/wal/instance:
15:27:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "367a150421b34b3493e473c47f1dcef9"
15:27:21.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:21 on dist-test-slave-fpc8"
15:27:21.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.428031 12679 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
15:27:21.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.433835 12700 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.435174 12679 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.000s
15:27:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.435457 12679 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/wal
15:27:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "367a150421b34b3493e473c47f1dcef9"
15:27:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:21 on dist-test-slave-fpc8"
15:27:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.435745 12679 fs_report.cc:383] FS layout report
15:27:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/wal
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/wal
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/data/data
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:21.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.502780 12679 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:21.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.504249 12679 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:21.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.504668 12679 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:21.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.575472 12679 rpc_server.cc:307] RPC server started. Bound to: 127.12.77.125:44219
15:27:21.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.575563 12751 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.77.125:44219 every 8 connection(s)
15:27:21.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.578006 12679 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/data/info.pb
15:27:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.582082 12597 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 12679
15:27:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.582542 12597 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-1/wal/instance
15:27:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.583741 12752 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:27:21.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.585577 12597 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/wal
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/data
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/logs
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/data/info.pb
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.124:33309
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.124
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:21.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37303/jwks.json with env {}
15:27:21.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.597966 12752 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } has no permanent_uuid. Determining permanent_uuid...
15:27:21.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.621276 12752 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } has no permanent_uuid. Determining permanent_uuid...
15:27:21.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.637979 12752 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } has no permanent_uuid. Determining permanent_uuid...
15:27:21.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.640614 12704 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:21.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.645475 12752 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.694909 12752 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 1
15:27:21.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.699203 12752 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:21.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.767654 12752 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 2
15:27:21.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.771668 12752 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:21.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.863108 12752 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 3
15:27:21.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.866755 12752 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.892717 12754 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.893401 12754 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:21.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.894078 12754 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.924655 12754 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.925024 12754 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.925330 12754 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.925688 12754 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.926010 12754 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.926270 12754 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37303/jwks.json
15:27:21.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.961328 12754 master_runner.cc:386] Master server non-default flags:
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/data
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/wal
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.124:33309
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37303/jwks.json
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/data/info.pb
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.124
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/logs
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:21.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.962945 12754 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.965036 12754 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:21.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.983319 12765 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:22.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.010176 12752 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 4
15:27:22.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.015098 12752 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:22.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.154697 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } attempt: 7
15:27:22.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.172214 12677 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } has no permanent_uuid. Determining permanent_uuid...
15:27:22.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.177240 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:22.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.235875 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 1
15:27:22.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.239987 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:22.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.314487 12752 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 5
15:27:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.318703 12752 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:22.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.321568 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 2
15:27:22.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.325220 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.410784 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 3
15:27:22.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.414770 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.555218 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 4
15:27:22.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.559401 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:22.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.842814 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 5
15:27:22.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.847601 12677 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:22.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:22.866149 12752 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 6
15:27:22.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:22.873282 12752 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.77.124:33309: Network error: Client connection negotiation failed: client connection to 127.12.77.124:33309: connect: Connection refused (error 111)
15:27:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.983803 12767 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:21.983326 12764 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:21.985585 12754 server_base.cc:1034] running on GCE node
15:27:23.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.181269 12754 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:23.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.184204 12754 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:23.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.185573 12754 hybrid_clock.cc:648] HybridClock initialized: now 1781969243185534 us; error 63 us; skew 500 ppm
15:27:23.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.186445 12754 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:23.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.193265 12754 webserver.cc:460] Webserver started at http://127.12.77.124:38475/ using document root <none> and password file <none>
15:27:23.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.194235 12754 fs_manager.cc:359] Metadata directory not provided
15:27:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.194465 12754 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:23.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.195029 12754 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.200210 12754 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/data/instance:
15:27:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16cdbe555bf3461289807658d9aee509"
15:27:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:23 on dist-test-slave-fpc8"
15:27:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.201458 12754 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/wal/instance:
15:27:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16cdbe555bf3461289807658d9aee509"
15:27:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:23 on dist-test-slave-fpc8"
15:27:23.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.209163 12754 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.004s
15:27:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.214671 12779 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.215759 12754 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
15:27:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.216110 12754 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/wal
15:27:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "16cdbe555bf3461289807658d9aee509"
15:27:23.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:23 on dist-test-slave-fpc8"
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.216478 12754 fs_report.cc:383] FS layout report
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/wal
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/wal
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/data/data
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:23.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:23.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.248908 12754 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.250430 12754 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:23.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.250928 12754 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.319344 12754 rpc_server.cc:307] RPC server started. Bound to: 127.12.77.124:33309
15:27:23.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.319415 12830 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.77.124:33309 every 8 connection(s)
15:27:23.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.322077 12754 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/data/info.pb
15:27:23.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.327015 12831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:27:23.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.327315 12597 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 12754
15:27:23.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.327786 12597 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/master-2/wal/instance
15:27:23.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.343240 12831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } has no permanent_uuid. Determining permanent_uuid...
15:27:23.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.370842 12831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } has no permanent_uuid. Determining permanent_uuid...
15:27:23.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.382885 12831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } has no permanent_uuid. Determining permanent_uuid...
15:27:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.399914 12831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509: Bootstrap starting.
15:27:23.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.399997 12677 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 6
15:27:23.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.406905 12831 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509: Neither blocks nor log segments found. Creating new log.
15:27:23.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.408751 12831 log.cc:824] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509: Log is configured to *not* fsync() on all Append() calls
15:27:23.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.414149 12831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509: No bootstrap required, opened a new log
15:27:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.423384 12677 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df: Bootstrap starting.
15:27:23.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.430266 12677 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df: Neither blocks nor log segments found. Creating new log.
15:27:23.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.432554 12677 log.cc:824] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df: Log is configured to *not* fsync() on all Append() calls
15:27:23.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.441599 12677 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df: No bootstrap required, opened a new log
15:27:23.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.442891 12831 raft_consensus.cc:348] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.443735 12831 raft_consensus.cc:374] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:27:23.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.444021 12831 raft_consensus.cc:729] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 16cdbe555bf3461289807658d9aee509, State: Initialized, Role: FOLLOWER
15:27:23.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.444902 12831 consensus_queue.cc:260] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.449818 12831 sys_catalog.cc:564] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: configured and running, proceeding with master startup.
15:27:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.448048 12842 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:23.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.451155 12842 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: This master's current role is: FOLLOWER
15:27:23.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.473731 12677 raft_consensus.cc:348] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:23.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.474956 12677 raft_consensus.cc:374] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:27:23.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.476477 12677 raft_consensus.cc:729] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f433500da0bf4d1295647e387a7bf1df, State: Initialized, Role: FOLLOWER
15:27:23.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.477726 12677 consensus_queue.cc:260] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:23.479096 12853 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:23.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:23.479604 12853 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.485539 12855 sys_catalog.cc:455] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.487120 12855 sys_catalog.cc:458] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [sys.catalog]: This master's current role is: FOLLOWER
15:27:23.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.487413 12677 sys_catalog.cc:564] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [sys.catalog]: configured and running, proceeding with master startup.
15:27:23.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:23.506387 12785 tablet.cc:2369] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:27:23.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.473589 12754 master_runner.cc:186] Error getting master registration for 127.12.77.126:35005: OK, instance_id { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" instance_seqno: 1781969239780879 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:27:23.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.511744 12754 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:27:23.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:23.526007 12866 catalog_manager.cc:1558] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:23.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:23.526644 12866 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:23.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.541970 12842 raft_consensus.cc:482] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:27:23.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.542420 12842 raft_consensus.cc:504] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.544488 12842 leader_election.cc:288] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f433500da0bf4d1295647e387a7bf1df (127.12.77.126:35005), 367a150421b34b3493e473c47f1dcef9 (127.12.77.125:44219)
15:27:23.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.545558 12652 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16cdbe555bf3461289807658d9aee509" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f433500da0bf4d1295647e387a7bf1df" is_pre_election: true
15:27:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.546433 12652 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 16cdbe555bf3461289807658d9aee509 in term 0.
15:27:23.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.547832 12782 leader_election.cc:302] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16cdbe555bf3461289807658d9aee509, f433500da0bf4d1295647e387a7bf1df; no voters:
15:27:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.548691 12842 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:27:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.549041 12842 raft_consensus.cc:482] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:27:23.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.549360 12842 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 0 FOLLOWER]: Advancing to term 1
15:27:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.555783 12842 raft_consensus.cc:504] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:23.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.559280 12652 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16cdbe555bf3461289807658d9aee509" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f433500da0bf4d1295647e387a7bf1df"
15:27:23.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.559846 12652 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [term 0 FOLLOWER]: Advancing to term 1
15:27:23.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.524395 12609 master_runner.cc:186] Error getting master registration for 127.12.77.125:44219: OK, instance_id { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" instance_seqno: 1781969241555365 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:27:23.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.564842 12609 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:27:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.567080 12652 raft_consensus.cc:2454] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 16cdbe555bf3461289807658d9aee509 in term 1.
15:27:23.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.567721 12842 leader_election.cc:288] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [CANDIDATE]: Term 1 election: Requested vote from peers f433500da0bf4d1295647e387a7bf1df (127.12.77.126:35005), 367a150421b34b3493e473c47f1dcef9 (127.12.77.125:44219)
15:27:23.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.570233 12782 leader_election.cc:302] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 16cdbe555bf3461289807658d9aee509, f433500da0bf4d1295647e387a7bf1df; no voters:
15:27:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.572268 12842 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 1 FOLLOWER]: Leader election won for term 1
15:27:23.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.585632 12842 raft_consensus.cc:686] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [term 1 LEADER]: Becoming Leader. State: Replica: 16cdbe555bf3461289807658d9aee509, State: Running, Role: LEADER
15:27:23.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.587097 12842 consensus_queue.cc:237] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.613206 12868 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16cdbe555bf3461289807658d9aee509. Latest consensus state: current_term: 1 leader_uuid: "16cdbe555bf3461289807658d9aee509" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:23.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.614425 12868 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: This master's current role is: LEADER
15:27:23.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.627704 12870 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
15:27:23.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.635172 12870 catalog_manager.cc:1484] Initializing Kudu cluster ID...
15:27:23.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.657389 12652 raft_consensus.cc:1261] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [term 1 FOLLOWER]: Refusing update from remote peer 16cdbe555bf3461289807658d9aee509: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:27:23.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.659571 12868 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:27:23.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.694655 12871 mvcc.cc:205] Tried to move back new op lower bound from 7298946021999788032 to 7298946021743976448. Current Snapshot: MvccSnapshot[applied={T|T < 7298946021999788032}]
15:27:23.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.697948 12855 sys_catalog.cc:455] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16cdbe555bf3461289807658d9aee509. Latest consensus state: current_term: 1 leader_uuid: "16cdbe555bf3461289807658d9aee509" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.701996 12842 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "16cdbe555bf3461289807658d9aee509" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.702704 12842 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: This master's current role is: LEADER
15:27:23.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.702039 12855 sys_catalog.cc:458] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [sys.catalog]: This master's current role is: FOLLOWER
15:27:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.705519 12870 catalog_manager.cc:1347] Generated new cluster ID: a55ab53445ea482ebff0e4f45d5fbca4
15:27:23.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.705789 12870 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
15:27:23.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.730262 12855 sys_catalog.cc:455] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "16cdbe555bf3461289807658d9aee509" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:23.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.732581 12855 sys_catalog.cc:458] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df [sys.catalog]: This master's current role is: FOLLOWER
15:27:23.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.758770 12870 catalog_manager.cc:1370] Generated new certificate authority record
15:27:23.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.762780 12870 catalog_manager.cc:1504] Loading token signing keys...
15:27:23.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.824615 12870 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509: Generated new TSK 0
15:27:23.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.836321 12870 catalog_manager.cc:1514] Initializing in-progress tserver states...
15:27:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.854279 12597 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/wal
15:27:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/data
15:27:23.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/logs
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/data/info.pb
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.65:0
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.77.65
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.65
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:27:23.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.933645 12752 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } attempt: 7
15:27:23.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:23.992221 12752 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9: Bootstrap starting.
15:27:24.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.014254 12752 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9: Neither blocks nor log segments found. Creating new log.
15:27:24.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.021248 12752 log.cc:824] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9: Log is configured to *not* fsync() on all Append() calls
15:27:24.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.041440 12752 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9: No bootstrap required, opened a new log
15:27:24.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.119396 12752 raft_consensus.cc:348] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.121958 12752 raft_consensus.cc:374] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:27:24.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.122959 12752 raft_consensus.cc:729] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 367a150421b34b3493e473c47f1dcef9, State: Initialized, Role: FOLLOWER
15:27:24.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.125370 12752 consensus_queue.cc:260] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } }
15:27:24.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.137269 12726 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16cdbe555bf3461289807658d9aee509" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "367a150421b34b3493e473c47f1dcef9"
15:27:24.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.138579 12727 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "16cdbe555bf3461289807658d9aee509" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "367a150421b34b3493e473c47f1dcef9" is_pre_election: true
15:27:24.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.140547 12881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:24.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.146754 12725 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [term 0 FOLLOWER]: Advancing to term 1
15:27:24.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.152376 12752 sys_catalog.cc:564] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [sys.catalog]: configured and running, proceeding with master startup.
15:27:24.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.159933 12881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [sys.catalog]: This master's current role is: FOLLOWER
15:27:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.163363 12725 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [term 1 FOLLOWER]: Refusing update from remote peer 16cdbe555bf3461289807658d9aee509: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:27:24.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.164965 12868 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [LEADER]: Connected to new peer: Peer: permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:27:24.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.220180 12881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 16cdbe555bf3461289807658d9aee509. Latest consensus state: current_term: 1 leader_uuid: "16cdbe555bf3461289807658d9aee509" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:24.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.220947 12881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [sys.catalog]: This master's current role is: FOLLOWER
15:27:24.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.222353 12886 sys_catalog.cc:455] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "16cdbe555bf3461289807658d9aee509" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:24.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.223337 12886 sys_catalog.cc:458] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9 [sys.catalog]: This master's current role is: FOLLOWER
15:27:24.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.277719 12897 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:27:24.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:24.405241 12877 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:24.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:24.405929 12877 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:24.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:24.406747 12877 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:24.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:24.448666 12877 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:24.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:24.449620 12877 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.77.65
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.490666 12877 tablet_server_runner.cc:78] Tablet server non-default flags:
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/data
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/wal
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.65:0
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/data/info.pb
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.65
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:24.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:24.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:24.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:24.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:24.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:24.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.77.65
15:27:24.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/logs
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.492210 12877 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:24.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.494378 12877 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:24.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:24.521131 12903 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:24.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.532492 12866 catalog_manager.cc:1259] Loaded cluster ID: a55ab53445ea482ebff0e4f45d5fbca4
15:27:24.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.533072 12866 catalog_manager.cc:1552] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df: loading cluster ID for follower catalog manager: success
15:27:24.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.540954 12866 catalog_manager.cc:1574] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df: acquiring CA information for follower catalog manager: success
15:27:24.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.555840 12866 catalog_manager.cc:1602] T 00000000000000000000000000000000 P f433500da0bf4d1295647e387a7bf1df: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:27:24.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.605144 12868 sys_catalog.cc:455] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "16cdbe555bf3461289807658d9aee509" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f433500da0bf4d1295647e387a7bf1df" member_type: VOTER last_known_addr { host: "127.12.77.126" port: 35005 } } peers { permanent_uuid: "367a150421b34b3493e473c47f1dcef9" member_type: VOTER last_known_addr { host: "127.12.77.125" port: 44219 } } peers { permanent_uuid: "16cdbe555bf3461289807658d9aee509" member_type: VOTER last_known_addr { host: "127.12.77.124" port: 33309 } } }
15:27:24.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:24.607174 12868 sys_catalog.cc:458] T 00000000000000000000000000000000 P 16cdbe555bf3461289807658d9aee509 [sys.catalog]: This master's current role is: LEADER
15:27:25.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:25.305459 12897 catalog_manager.cc:1259] Loaded cluster ID: a55ab53445ea482ebff0e4f45d5fbca4
15:27:25.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:25.305743 12897 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9: loading cluster ID for follower catalog manager: success
15:27:25.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:25.320261 12897 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9: acquiring CA information for follower catalog manager: success
15:27:25.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:25.325822 12897 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 367a150421b34b3493e473c47f1dcef9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:27:27.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:25.913800 12902 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 12877
15:27:27.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:27.573817 12902 kernel_stack_watchdog.cc:198] Thread 12877 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 390ms:
15:27:27.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:27:27.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:27:27.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:27:27.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:27:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:27.574666 12877 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 3.054s user 1.020s sys 1.780s
15:27:27.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:27.575368 12877 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 3.054s user 1.020s sys 1.780s
15:27:27.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:27.576434 12906 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:27.581060 12905 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3052 milliseconds
15:27:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:24.527896 12904 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:27.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.581626 12877 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:27:27.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.583037 12877 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:27.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.595568 12877 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.603858 12877 hybrid_clock.cc:648] HybridClock initialized: now 1781969247603820 us; error 66 us; skew 500 ppm
15:27:27.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.604914 12877 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.616611 12877 webserver.cc:460] Webserver started at http://127.12.77.65:40985/ using document root <none> and password file <none>
15:27:27.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.617662 12877 fs_manager.cc:359] Metadata directory not provided
15:27:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.617920 12877 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.618450 12877 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.623610 12877 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/data/instance:
15:27:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e5e586553424625ba36db5ef0b36a43"
15:27:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:27 on dist-test-slave-fpc8"
15:27:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.624854 12877 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/wal/instance:
15:27:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e5e586553424625ba36db5ef0b36a43"
15:27:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:27 on dist-test-slave-fpc8"
15:27:27.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.634025 12877 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
15:27:27.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.641489 12919 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.642829 12877 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.004s sys 0.002s
15:27:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.643256 12877 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/wal
15:27:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e5e586553424625ba36db5ef0b36a43"
15:27:27.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:27 on dist-test-slave-fpc8"
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.643700 12877 fs_report.cc:383] FS layout report
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/wal
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/wal
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/data/data
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:27.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:27.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:27.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:27.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.697719 12877 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:27.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.699740 12877 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:27.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.700335 12877 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:27.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.703809 12877 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:27:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.709921 12877 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:27:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.710201 12877 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:27:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.710496 12877 ts_tablet_manager.cc:610] Registered 0 tablets
15:27:27.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:27.710726 12877 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:27:28.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.006946 12877 rpc_server.cc:307] RPC server started. Bound to: 127.12.77.65:46455
15:27:28.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.007062 13032 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.77.65:46455 every 8 connection(s)
15:27:28.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.010049 12877 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/data/info.pb
15:27:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.020934 12597 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 12877
15:27:28.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.021770 12597 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-0/wal/instance
15:27:28.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.042541 12597 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:28.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:28.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/wal
15:27:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/data
15:27:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:28.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/logs
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/data/info.pb
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.66:0
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.77.66
15:27:28.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.66
15:27:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:27:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.107905 13033 heartbeater.cc:347] Connected to a master server at 127.12.77.124:33309
15:27:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.108461 13033 heartbeater.cc:464] Registering TS with master...
15:27:28.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.109776 13033 heartbeater.cc:511] Master 127.12.77.124:33309 requested a full tablet report, sending...
15:27:28.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.112914 12796 ts_manager.cc:194] Registered new tserver with Master: 9e5e586553424625ba36db5ef0b36a43 (127.12.77.65:46455)
15:27:28.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.115662 12796 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.77.65:35559
15:27:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.121165 13034 heartbeater.cc:347] Connected to a master server at 127.12.77.125:44219
15:27:28.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.121500 13034 heartbeater.cc:464] Registering TS with master...
15:27:28.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.122179 13034 heartbeater.cc:511] Master 127.12.77.125:44219 requested a full tablet report, sending...
15:27:28.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.125268 12717 ts_manager.cc:194] Registered new tserver with Master: 9e5e586553424625ba36db5ef0b36a43 (127.12.77.65:46455)
15:27:28.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.132349 13038 heartbeater.cc:347] Connected to a master server at 127.12.77.126:35005
15:27:28.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.132716 13038 heartbeater.cc:464] Registering TS with master...
15:27:28.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.133378 13038 heartbeater.cc:511] Master 127.12.77.126:35005 requested a full tablet report, sending...
15:27:28.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.136660 12642 ts_manager.cc:194] Registered new tserver with Master: 9e5e586553424625ba36db5ef0b36a43 (127.12.77.65:46455)
15:27:28.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:28.439466 13037 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:28.440019 13037 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:28.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:28.440639 13037 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:28.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:28.472868 13037 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:28.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:28.473891 13037 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.77.66
15:27:28.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.513949 13037 tablet_server_runner.cc:78] Tablet server non-default flags:
15:27:28.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:28.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:28.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:28.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:28.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/data
15:27:28.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/wal
15:27:28.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.66:0
15:27:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/data/info.pb
15:27:28.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.66
15:27:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:28.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.77.66
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/logs
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.515504 13037 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:28.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:28.517536 13037 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:28.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:28.541776 13051 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:29.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.121026 13033 heartbeater.cc:503] Master 127.12.77.124:33309 was elected leader, sending a full tablet report...
15:27:29.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:28.543306 13050 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:28.541834 13053 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:29.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:29.950695 13052 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1408 milliseconds
15:27:29.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.950870 13037 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:27:29.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.952333 13037 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:29.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.955086 13037 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:29.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.956530 13037 hybrid_clock.cc:648] HybridClock initialized: now 1781969249956486 us; error 62 us; skew 500 ppm
15:27:29.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.957458 13037 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:29.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.964458 13037 webserver.cc:460] Webserver started at http://127.12.77.66:38407/ using document root <none> and password file <none>
15:27:29.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.965334 13037 fs_manager.cc:359] Metadata directory not provided
15:27:29.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.965525 13037 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:29.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.965905 13037 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:29.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.971086 13037 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/data/instance:
15:27:29.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c040ca44e144f3eabbf9776d2701e20"
15:27:29.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:29 on dist-test-slave-fpc8"
15:27:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.972153 13037 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/wal/instance:
15:27:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c040ca44e144f3eabbf9776d2701e20"
15:27:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:29 on dist-test-slave-fpc8"
15:27:29.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.979509 13037 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
15:27:29.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.985103 13060 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:29.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.986142 13037 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:27:29.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.986501 13037 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/wal
15:27:29.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0c040ca44e144f3eabbf9776d2701e20"
15:27:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:29 on dist-test-slave-fpc8"
15:27:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:29.986840 13037 fs_report.cc:383] FS layout report
15:27:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/wal
15:27:29.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/wal
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/data/data
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:29.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:30.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.042824 13037 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:30.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.044377 13037 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.044890 13037 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.047418 13037 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:27:30.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.051694 13037 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:27:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.051925 13037 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:27:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.052260 13037 ts_tablet_manager.cc:610] Registered 0 tablets
15:27:30.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.052444 13037 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:27:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.199174 13037 rpc_server.cc:307] RPC server started. Bound to: 127.12.77.66:39273
15:27:30.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.199291 13173 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.77.66:39273 every 8 connection(s)
15:27:30.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.201746 13037 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/data/info.pb
15:27:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.203852 12597 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 13037
15:27:30.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.204427 12597 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-1/wal/instance
15:27:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.231433 12597 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/wal
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/data
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/logs
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/data/info.pb
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:30.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.67:0
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.77.67
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.67
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:27:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.266662 13174 heartbeater.cc:347] Connected to a master server at 127.12.77.124:33309
15:27:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.267283 13175 heartbeater.cc:347] Connected to a master server at 127.12.77.125:44219
15:27:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.267668 13174 heartbeater.cc:464] Registering TS with master...
15:27:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.267701 13175 heartbeater.cc:464] Registering TS with master...
15:27:30.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.268970 13175 heartbeater.cc:511] Master 127.12.77.125:44219 requested a full tablet report, sending...
15:27:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.270951 13174 heartbeater.cc:511] Master 127.12.77.124:33309 requested a full tablet report, sending...
15:27:30.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.272540 12717 ts_manager.cc:194] Registered new tserver with Master: 0c040ca44e144f3eabbf9776d2701e20 (127.12.77.66:39273)
15:27:30.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.273375 12796 ts_manager.cc:194] Registered new tserver with Master: 0c040ca44e144f3eabbf9776d2701e20 (127.12.77.66:39273)
15:27:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.275410 12796 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.77.66:50677
15:27:30.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.276862 13176 heartbeater.cc:347] Connected to a master server at 127.12.77.126:35005
15:27:30.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.277180 13176 heartbeater.cc:464] Registering TS with master...
15:27:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.277887 13176 heartbeater.cc:511] Master 127.12.77.126:35005 requested a full tablet report, sending...
15:27:30.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.280452 12642 ts_manager.cc:194] Registered new tserver with Master: 0c040ca44e144f3eabbf9776d2701e20 (127.12.77.66:39273)
15:27:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:30.564594 13182 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:30.565073 13182 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:30.565491 13182 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:30.598649 13182 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:30.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:30.599622 13182 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.77.67
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.633989 13182 tablet_server_runner.cc:78] Tablet server non-default flags:
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.77.84:34151
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/data
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/wal
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.77.67:0
15:27:30.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/data/info.pb
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.77.67
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.77.126:35005,127.12.77.125:44219,127.12.77.124:33309
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.77.67
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/logs
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:30.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.635457 13182 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:30.637310 13182 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:30.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:30.655579 13191 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:31.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.279631 13174 heartbeater.cc:503] Master 127.12.77.124:33309 was elected leader, sending a full tablet report...
15:27:31.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:30.656424 13190 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:31.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:30.655620 13193 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:31.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:31.850150 13192 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:27:31.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.850587 13182 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:27:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.854900 13182 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.857299 13182 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.858623 13182 hybrid_clock.cc:648] HybridClock initialized: now 1781969251858600 us; error 66 us; skew 500 ppm
15:27:31.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.859509 13182 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:31.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.866073 13182 webserver.cc:460] Webserver started at http://127.12.77.67:35843/ using document root <none> and password file <none>
15:27:31.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.867020 13182 fs_manager.cc:359] Metadata directory not provided
15:27:31.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.867234 13182 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:31.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.867626 13182 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.872287 13182 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/data/instance:
15:27:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c05e339ba4434007b09a4121bf4ebb19"
15:27:31.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:31 on dist-test-slave-fpc8"
15:27:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.873334 13182 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/wal/instance:
15:27:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c05e339ba4434007b09a4121bf4ebb19"
15:27:31.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:31 on dist-test-slave-fpc8"
15:27:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.881930 13182 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
15:27:31.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.891218 13201 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:31.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.892480 13182 fs_manager.cc:725] Time spent opening block manager: real 0.006s user 0.007s sys 0.000s
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.892865 13182 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/wal
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c05e339ba4434007b09a4121bf4ebb19"
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:31 on dist-test-slave-fpc8"
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.893208 13182 fs_report.cc:383] FS layout report
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/wal
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/wal
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/data/data
15:27:31.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:31.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.953807 13182 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:31.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.955478 13182 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:31.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.955909 13182 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:31.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.958904 13182 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
15:27:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.963274 13182 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:27:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.963485 13182 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:27:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.963677 13182 ts_tablet_manager.cc:610] Registered 0 tablets
15:27:31.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:31.963810 13182 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:27:32.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.108858 13182 rpc_server.cc:307] RPC server started. Bound to: 127.12.77.67:37275
15:27:32.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.108984 13313 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.12.77.67:37275 every 8 connection(s)
15:27:32.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.111356 13182 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/data/info.pb
15:27:32.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.118355 12597 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 13182
15:27:32.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.118821 12597 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster1390894222240788911/ts-2/wal/instance
15:27:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.161536 13315 heartbeater.cc:347] Connected to a master server at 127.12.77.125:44219
15:27:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.161973 13315 heartbeater.cc:464] Registering TS with master...
15:27:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.162600 13314 heartbeater.cc:347] Connected to a master server at 127.12.77.124:33309
15:27:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.162954 13314 heartbeater.cc:464] Registering TS with master...
15:27:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.163247 13315 heartbeater.cc:511] Master 127.12.77.125:44219 requested a full tablet report, sending...
15:27:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.163594 13314 heartbeater.cc:511] Master 127.12.77.124:33309 requested a full tablet report, sending...
15:27:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.163724 13316 heartbeater.cc:347] Connected to a master server at 127.12.77.126:35005
15:27:32.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.164044 13316 heartbeater.cc:464] Registering TS with master...
15:27:32.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.164847 13316 heartbeater.cc:511] Master 127.12.77.126:35005 requested a full tablet report, sending...
15:27:32.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.166066 12795 ts_manager.cc:194] Registered new tserver with Master: c05e339ba4434007b09a4121bf4ebb19 (127.12.77.67:37275)
15:27:32.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.166522 12717 ts_manager.cc:194] Registered new tserver with Master: c05e339ba4434007b09a4121bf4ebb19 (127.12.77.67:37275)
15:27:32.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.168147 12795 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.77.67:34103
15:27:32.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.168829 12642 ts_manager.cc:194] Registered new tserver with Master: c05e339ba4434007b09a4121bf4ebb19 (127.12.77.67:37275)
15:27:32.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.186064 12597 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
15:27:32.197 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:27:32.198 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:27:32.209 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.77.126"
port: 35005
}
bound_http_address {
host: "127.12.77.126"
port: 40845
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.77.125"
port: 44219
}
bound_http_address {
host: "127.12.77.125"
port: 39967
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.77.124"
port: 33309
}
bound_http_address {
host: "127.12.77.124"
port: 38475
}
}
}
15:27:32.214 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:27:32.217 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.77.65"
port: 46455
}
bound_http_address {
host: "127.12.77.65"
port: 40985
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.77.66"
port: 39273
}
bound_http_address {
host: "127.12.77.66"
port: 38407
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.77.67"
port: 37275
}
bound_http_address {
host: "127.12.77.67"
port: 35843
}
}
}
15:27:32.217 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:27:32.409 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:27:32.532 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNRqU_n/test-tmp
15:27:32.533 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:27:32.533 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNRqU_n/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:27:32.536 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:27:32.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:32.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:27:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.876552 13325 mini_oidc.cc:152] Starting JWKS server
15:27:32.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.880125 13325 webserver.cc:460] Webserver started at http://127.0.0.1:46455/ using document root <none> and password file <none>
15:27:32.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.880515 13325 mini_oidc.cc:189] Starting OIDC Discovery server
15:27:32.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:32.882225 13325 webserver.cc:460] Webserver started at http://127.0.0.1:42485/ using document root <none> and password file <none>
15:27:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:27:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/principal' for realm 'KRBTEST.COM',
15:27:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:27:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:32 dist-test-slave-fpc8 krb5kdc[13337](info): setting up network...
15:27:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:27:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:32 dist-test-slave-fpc8 krb5kdc[13337](info): set up 2 sockets
15:27:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:32 dist-test-slave-fpc8 krb5kdc[13337](info): commencing operation
15:27:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:27:33.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:33.005909 13029 messenger.cc:188] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.12.77.65:46455
15:27:33.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:33.172053 13314 heartbeater.cc:503] Master 127.12.77.124:33309 was elected leader, sending a full tablet report...
15:27:33.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:33.997082 13325 mini_kdc.cc:121] Time spent starting KDC: real 1.091s user 0.008s sys 0.011s
15:27:34.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:27:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:27:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:27:34.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:27:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:27:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:27:34.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:27:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:27:34.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:27:34.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:27:34.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/test-user.keytab.
15:27:34.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/test-user.keytab.
15:27:34.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:34 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969254, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:34.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:27:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:27:34Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
15:27:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:27:34Z Disabled control of system clock
15:27:34.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.3.126@KRBTEST.COM; defaulting to no policy
15:27:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:27:34.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.3.126@KRBTEST.COM" created.
15:27:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:27:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.3.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.126.keytab.
15:27:34.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.3.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.126.keytab.
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:34.187928 13325 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/logs
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/info.pb
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:34.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.126:39741
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.126
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.126.keytab
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.126
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:34.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:34.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:27:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.501004 13350 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.501646 13350 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:34.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.502316 13350 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.534682 13350 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:27:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.535030 13350 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:34.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.535472 13350 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.535794 13350 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.536280 13350 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:34.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.536634 13350 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:34.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.536885 13350 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46455/jwks.json
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:34.578580 13350 master_runner.cc:386] Master server non-default flags:
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.126.keytab
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.126
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.126:39741
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/info.pb
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.126
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/logs
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:34.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:34.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:34.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:34.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:34.580024 13350 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:34.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:34.581302 13350 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:34.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.601025 13358 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:35.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.601397 13360 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:35.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:34.603565 13357 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:35.893903 13359 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1291 milliseconds
15:27:35.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.894039 13350 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:27:35.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.895357 13350 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:35.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.897714 13350 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:35.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.899063 13350 hybrid_clock.cc:648] HybridClock initialized: now 1781969255899040 us; error 39 us; skew 500 ppm
15:27:35.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.899868 13350 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:35.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:35 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969255, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:35.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.906211 13350 init.cc:377] Logged in from keytab as kudu/127.13.3.126@KRBTEST.COM (short username kudu)
15:27:35.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.911978 13350 webserver.cc:460] Webserver started at http://127.13.3.126:44265/ using document root <none> and password file <none>
15:27:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.912968 13350 fs_manager.cc:359] Metadata directory not provided
15:27:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.913149 13350 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:35.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.913519 13350 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:35.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.918311 13350 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/instance:
15:27:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ae1821a22154b54ba0e6ee844209349"
15:27:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:35 on dist-test-slave-fpc8"
15:27:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.919502 13350 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal/instance:
15:27:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ae1821a22154b54ba0e6ee844209349"
15:27:35.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:35 on dist-test-slave-fpc8"
15:27:35.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.927443 13350 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
15:27:35.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.933734 13368 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:35.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.935057 13350 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
15:27:35.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.935367 13350 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:35.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ae1821a22154b54ba0e6ee844209349"
15:27:35.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:35 on dist-test-slave-fpc8"
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.935730 13350 fs_report.cc:383] FS layout report
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/data
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:35.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:35.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.984755 13350 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:35.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.990270 13350 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:35.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:35.990775 13350 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.068651 13350 rpc_server.cc:307] RPC server started. Bound to: 127.13.3.126:39741
15:27:36.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.068672 13419 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.13.3.126:39741 every 8 connection(s)
15:27:36.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.071916 13350 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/info.pb
15:27:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.074057 13325 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 13350
15:27:36.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.074608 13325 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal/instance
15:27:36.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.078675 13420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:27:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.3.125@KRBTEST.COM; defaulting to no policy
15:27:36.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.092744 13420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } has no permanent_uuid. Determining permanent_uuid...
15:27:36.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:27:36.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.3.125@KRBTEST.COM" created.
15:27:36.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:36 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969255, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.126@KRBTEST.COM for kudu/127.13.3.126@KRBTEST.COM
15:27:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:27:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.3.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.125.keytab.
15:27:36.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.3.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.125.keytab.
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.124748 13325 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/logs
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/info.pb
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.125:35307
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.125
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.125.keytab
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.125
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:27:36.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.136526 13420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } has no permanent_uuid. Determining permanent_uuid...
15:27:36.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.139052 13370 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:36.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.143620 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.125:35307: Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:36.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.193136 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } attempt: 1
15:27:36.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.197369 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.125:35307: Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:36.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.265810 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } attempt: 2
15:27:36.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.269779 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.125:35307: Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:36.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.361208 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } attempt: 3
15:27:36.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.365470 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.125:35307: Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:36.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.443041 13426 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:36.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.443574 13426 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:36.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.444125 13426 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.474819 13426 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:27:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.475502 13426 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.475849 13426 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.476051 13426 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.476236 13426 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.476428 13426 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.476564 13426 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46455/jwks.json
15:27:36.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.508885 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } attempt: 4
15:27:36.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.512496 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.125:35307: Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.519312 13426 master_runner.cc:386] Master server non-default flags:
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.125.keytab
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.125
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.125:35307
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/info.pb
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.125
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:36.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/logs
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.520411 13426 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:36.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.521495 13426 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.540992 13434 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:36.811954 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } attempt: 5
15:27:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.817099 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.125:35307: Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:37.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.364653 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } attempt: 6
15:27:37.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:37.371918 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.125:35307: Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.541793 13435 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:37.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:36.541016 13437 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:37.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:37.769603 13436 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:27:37.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.769845 13426 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:27:37.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.774008 13426 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:37.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.777267 13426 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:37.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.778692 13426 hybrid_clock.cc:648] HybridClock initialized: now 1781969257778635 us; error 79 us; skew 500 ppm
15:27:37.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.779522 13426 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:37.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:37 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969257, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.785584 13426 init.cc:377] Logged in from keytab as kudu/127.13.3.125@KRBTEST.COM (short username kudu)
15:27:37.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.791175 13426 webserver.cc:460] Webserver started at http://127.13.3.125:42515/ using document root <none> and password file <none>
15:27:37.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.792027 13426 fs_manager.cc:359] Metadata directory not provided
15:27:37.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.792200 13426 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:37.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.792572 13426 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:37.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.797932 13426 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/instance:
15:27:37.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eacd7ab7198468f9093b33ce8018ebb"
15:27:37.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:37 on dist-test-slave-fpc8"
15:27:37.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.799031 13426 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal/instance:
15:27:37.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eacd7ab7198468f9093b33ce8018ebb"
15:27:37.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:37 on dist-test-slave-fpc8"
15:27:37.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.806327 13426 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
15:27:37.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.813571 13449 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:37.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.814658 13426 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.003s
15:27:37.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.814978 13426 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:37.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eacd7ab7198468f9093b33ce8018ebb"
15:27:37.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:37 on dist-test-slave-fpc8"
15:27:37.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.815268 13426 fs_report.cc:383] FS layout report
15:27:37.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/data
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:37.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:37.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.864383 13426 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:37.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.869443 13426 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:37.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.869824 13426 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:37.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.954535 13426 rpc_server.cc:307] RPC server started. Bound to: 127.13.3.125:35307
15:27:37.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.954700 13501 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.13.3.125:35307 every 8 connection(s)
15:27:37.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.957805 13426 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/info.pb
15:27:37.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.964823 13502 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:27:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.968259 13325 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 13426
15:27:37.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.968708 13325 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal/instance
15:27:37.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:37.981473 13502 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } has no permanent_uuid. Determining permanent_uuid...
15:27:37.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.3.124@KRBTEST.COM; defaulting to no policy
15:27:37.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:27:37.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.3.124@KRBTEST.COM" created.
15:27:38.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:38 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969257, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.125@KRBTEST.COM for kudu/127.13.3.126@KRBTEST.COM
15:27:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:27:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.3.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.124.keytab.
15:27:38.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.3.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.124.keytab.
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.016921 13325 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/logs
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/info.pb
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.124:34635
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.124
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.124.keytab
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.124
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:27:38.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.024636 13502 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } has no permanent_uuid. Determining permanent_uuid...
15:27:38.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:38 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969257, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.125@KRBTEST.COM for kudu/127.13.3.125@KRBTEST.COM
15:27:38.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.054184 13502 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } has no permanent_uuid. Determining permanent_uuid...
15:27:38.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.057047 13452 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.061331 13502 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.110769 13502 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 1
15:27:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.115096 13502 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.183490 13502 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 2
15:27:38.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.187565 13502 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.279026 13502 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 3
15:27:38.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.283142 13502 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.360417 13508 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:38.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.360937 13508 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:38.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.361531 13508 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.394438 13508 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:27:38.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.394656 13508 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.395004 13508 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.395219 13508 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.395411 13508 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.395596 13508 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.395732 13508 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46455/jwks.json
15:27:38.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.426555 13502 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 4
15:27:38.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.431110 13502 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.432484 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } attempt: 7
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.436944 13508 master_runner.cc:386] Master server non-default flags:
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.124.keytab
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.124
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.124:34635
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:38.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/info.pb
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.124
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/logs
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:38.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.438422 13508 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:38.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.439564 13508 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:38.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:38 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969255, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.126@KRBTEST.COM for kudu/127.13.3.125@KRBTEST.COM
15:27:38.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.465796 13520 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:38.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.474151 13420 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } has no permanent_uuid. Determining permanent_uuid...
15:27:38.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.480837 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.539278 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 1
15:27:38.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.544957 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.626463 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 2
15:27:38.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.634693 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.720212 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 3
15:27:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.726019 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.730851 13502 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 5
15:27:38.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.737547 13502 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:38.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:38.866513 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 4
15:27:38.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.882373 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:39.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.167019 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 5
15:27:39.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:39.175299 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:39.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.285095 13502 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 6
15:27:39.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:39.294783 13502 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:39.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.727856 13420 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } attempt: 6
15:27:39.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:39.736557 13420 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.3.124:34635: Network error: Client connection negotiation failed: client connection to 127.13.3.124:34635: connect: Connection refused (error 111)
15:27:39.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.465763 13519 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:39.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:38.467160 13522 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:39.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:39.780604 13521 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1313 milliseconds
15:27:39.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.780711 13508 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:27:39.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.781884 13508 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:39.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.785008 13508 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:39.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.786430 13508 hybrid_clock.cc:648] HybridClock initialized: now 1781969259786388 us; error 62 us; skew 500 ppm
15:27:39.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.787573 13508 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:39.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:39 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969259, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:39.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.793934 13508 init.cc:377] Logged in from keytab as kudu/127.13.3.124@KRBTEST.COM (short username kudu)
15:27:39.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.800026 13508 webserver.cc:460] Webserver started at http://127.13.3.124:46329/ using document root <none> and password file <none>
15:27:39.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.800944 13508 fs_manager.cc:359] Metadata directory not provided
15:27:39.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.801132 13508 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:39.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.801535 13508 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
15:27:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.807212 13508 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/instance:
15:27:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e267919edda9441493579257acc2e98b"
15:27:39.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:39 on dist-test-slave-fpc8"
15:27:39.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.808633 13508 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal/instance:
15:27:39.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e267919edda9441493579257acc2e98b"
15:27:39.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:39 on dist-test-slave-fpc8"
15:27:39.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.817458 13508 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
15:27:39.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.824776 13536 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:39.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.825996 13508 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:27:39.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.826351 13508 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:39.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e267919edda9441493579257acc2e98b"
15:27:39.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:39 on dist-test-slave-fpc8"
15:27:39.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.826756 13508 fs_report.cc:383] FS layout report
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/data
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:39.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:39.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.873955 13508 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.880055 13508 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:39.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.880481 13508 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.986876 13508 rpc_server.cc:307] RPC server started. Bound to: 127.13.3.124:34635
15:27:39.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.987093 13587 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.13.3.124:34635 every 8 connection(s)
15:27:39.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.989838 13508 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/info.pb
15:27:39.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.995677 13588 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:27:39.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.997812 13325 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 13508
15:27:39.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:39.998301 13325 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal/instance
15:27:40.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.013305 13588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } has no permanent_uuid. Determining permanent_uuid...
15:27:40.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:40 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969254, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.3.124@KRBTEST.COM
15:27:40.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:40 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969259, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.124@KRBTEST.COM for kudu/127.13.3.126@KRBTEST.COM
15:27:40.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.050144 13588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } has no permanent_uuid. Determining permanent_uuid...
15:27:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:40 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969259, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.124@KRBTEST.COM for kudu/127.13.3.125@KRBTEST.COM
15:27:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:40 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969254, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.3.125@KRBTEST.COM
15:27:40.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.078281 13588 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } has no permanent_uuid. Determining permanent_uuid...
15:27:40.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:40 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969259, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.124@KRBTEST.COM for kudu/127.13.3.124@KRBTEST.COM
15:27:40.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:40 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969254, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.3.126@KRBTEST.COM
15:27:40.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.108925 13588 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: Bootstrap starting.
15:27:40.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.117056 13588 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: Neither blocks nor log segments found. Creating new log.
15:27:40.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.117836 13325 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
15:27:40.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.119127 13588 log.cc:824] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: Log is configured to *not* fsync() on all Append() calls
15:27:40.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.124469 13588 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: No bootstrap required, opened a new log
15:27:40.125 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:40.127 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:27:40.137 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:27:40.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:27:40.141 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.13.3.126"
port: 39741
}
bound_http_address {
host: "127.13.3.126"
port: 44265
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.13.3.125"
port: 35307
}
bound_http_address {
host: "127.13.3.125"
port: 42515
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.13.3.124"
port: 34635
}
bound_http_address {
host: "127.13.3.124"
port: 46329
}
}
}
15:27:40.142 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:27:40.142 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:40.142 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:27:40.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:27:40.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.147601 13588 raft_consensus.cc:348] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:40.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.148490 13588 raft_consensus.cc:374] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:27:40.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.149297 13588 raft_consensus.cc:729] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e267919edda9441493579257acc2e98b, State: Initialized, Role: FOLLOWER
15:27:40.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.150573 13588 consensus_queue.cc:260] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:40.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.164330 13599 sys_catalog.cc:455] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } } }
15:27:40.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.167102 13588 sys_catalog.cc:564] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [sys.catalog]: configured and running, proceeding with master startup.
15:27:40.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.167246 13599 sys_catalog.cc:458] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [sys.catalog]: This master's current role is: FOLLOWER
15:27:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:40 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969260, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:40.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:27:40.188 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:40.190 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc
15:27:40.210 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:27:40.212 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc
15:27:40.214 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:27:40.218 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:27:40.221 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:27:40.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.221294 13611 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:40.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:27:40.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.221676 13611 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:40.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.199239 13508 master_runner.cc:186] Error getting master registration for 127.13.3.126:39741: OK, instance_id { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" instance_seqno: 1781969256048267 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:27:40.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.229998 13508 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:27:40.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:40 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969260, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:40.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:27:40.260 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:40.260 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc
15:27:40.263 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:27:40.263 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.3.125:35307
15:27:40.265 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:27:40.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.266223 13325 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 13426
15:27:40.291 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:40.292 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.3.124:34635
15:27:40.292 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:27:40.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.292675 13325 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 13508
15:27:40.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:40.321 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.3.126:39741
15:27:40.321 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:27:40.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.321820 13325 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 13350
15:27:40.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:40.346 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.3.125:35307
15:27:40.348 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:27:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.350102 13325 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data
15:27:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:40.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/logs
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/info.pb
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.125:35307
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.125
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42515
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.125.keytab
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.125
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:27:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:40.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:27:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.676353 13615 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:40.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.677039 13615 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:40.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.677788 13615 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:40.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.710515 13615 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:27:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.710786 13615 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.711220 13615 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:40.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.711561 13615 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:40.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.711879 13615 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:40.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.712181 13615 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:40.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.712401 13615 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46455/jwks.json
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.749599 13615 master_runner.cc:386] Master server non-default flags:
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.125.keytab
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.125
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.125:35307
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:27:40.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/info.pb
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.125
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42515
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/logs
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:40.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.751151 13615 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:40.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.752305 13615 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:40.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.770901 13622 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:42.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.771063 13623 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:42.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:40.771544 13625 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:42.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:40.772651 13615 server_base.cc:1034] running on GCE node
15:27:42.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.029862 13615 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:42.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.032459 13615 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.033825 13615 hybrid_clock.cc:648] HybridClock initialized: now 1781969262033782 us; error 53 us; skew 500 ppm
15:27:42.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.034726 13615 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:42.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:42 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969262, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:42.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.040407 13615 init.cc:377] Logged in from keytab as kudu/127.13.3.125@KRBTEST.COM (short username kudu)
15:27:42.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.045492 13615 webserver.cc:460] Webserver started at http://127.13.3.125:42515/ using document root <none> and password file <none>
15:27:42.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.046402 13615 fs_manager.cc:359] Metadata directory not provided
15:27:42.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.046581 13615 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:42.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.054270 13615 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.006s sys 0.000s
15:27:42.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.058589 13634 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:42.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.059778 13615 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.060039 13615 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4eacd7ab7198468f9093b33ce8018ebb"
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:37 on dist-test-slave-fpc8"
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.062227 13615 fs_report.cc:383] FS layout report
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/data
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:42.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:42.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:42.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:42.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.095299 13615 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:42.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.100629 13615 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:42.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.101011 13615 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:42.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.173759 13615 rpc_server.cc:307] RPC server started. Bound to: 127.13.3.125:35307
15:27:42.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.173884 13685 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.13.3.125:35307 every 8 connection(s)
15:27:42.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.176551 13615 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/data/info.pb
15:27:42.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.183355 13325 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 13615
15:27:42.184 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:42.184 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.3.124:34635
15:27:42.184 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:27:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.186319 13325 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data
15:27:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:42.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/logs
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/info.pb
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:42.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.124:34635
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.124
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46329
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.124.keytab
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.124
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:42.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:27:42.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.190996 13686 sys_catalog.cc:263] Verifying existing consensus state
15:27:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 15:27:42.191982 13686 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:27:42.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.192458 13615 master.cc:537] Master@127.13.3.125:35307 shutting down...
15:27:42.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.256839 13615 master.cc:552] Master@127.13.3.125:35307 shutdown complete.
15:27:42.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:27:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.506676 13688 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:42.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.507347 13688 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:42.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.508021 13688 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:42.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.538582 13688 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:27:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.538793 13688 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.539239 13688 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:42.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.539578 13688 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:42.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.539891 13688 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:42.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.540215 13688 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:42.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.540450 13688 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46455/jwks.json
15:27:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.577406 13688 master_runner.cc:386] Master server non-default flags:
15:27:42.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.124.keytab
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.124
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.124:34635
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:27:42.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/info.pb
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.124
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46329
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:42.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/logs
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.578887 13688 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:42.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.580212 13688 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:42.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.598453 13695 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:42.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.902582 13029 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba7c0 after lost signal to thread 12898
15:27:42.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.903519 13029 debug-util.cc:398] Leaking SignalData structure 0x7b08000afe60 after lost signal to thread 13032
15:27:43.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.600811 13697 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:43.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:42.601233 13688 server_base.cc:1034] running on GCE node
15:27:43.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:42.598691 13694 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:43.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.874656 13688 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:43.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.878033 13688 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:43.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.879631 13688 hybrid_clock.cc:648] HybridClock initialized: now 1781969263879590 us; error 47 us; skew 500 ppm
15:27:43.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.880824 13688 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:43.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:43 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969263, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:43.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.892038 13688 init.cc:377] Logged in from keytab as kudu/127.13.3.124@KRBTEST.COM (short username kudu)
15:27:43.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.898226 13688 webserver.cc:460] Webserver started at http://127.13.3.124:46329/ using document root <none> and password file <none>
15:27:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.899503 13688 fs_manager.cc:359] Metadata directory not provided
15:27:43.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.899783 13688 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.911332 13688 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.007s sys 0.000s
15:27:43.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.917013 13707 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:43.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.918345 13688 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:27:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.918759 13688 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e267919edda9441493579257acc2e98b"
15:27:43.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:39 on dist-test-slave-fpc8"
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.922304 13688 fs_report.cc:383] FS layout report
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/wal
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/data
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:43.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:43.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.980887 13688 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:43.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.989365 13688 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:43.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:43.989928 13688 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:44.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.078405 13688 rpc_server.cc:307] RPC server started. Bound to: 127.13.3.124:34635
15:27:44.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.078483 13758 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.13.3.124:34635 every 8 connection(s)
15:27:44.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.082437 13688 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-2/data/info.pb
15:27:44.086 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:44.086 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.3.126:39741
15:27:44.086 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:27:44.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.084167 13325 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 13688
15:27:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.089991 13325 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu
15:27:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data
15:27:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:27:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:27:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:27:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/logs
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/info.pb
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.126:39741
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.126
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44265
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.126.keytab
15:27:44.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.126
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:44.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:27:44.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.100759 13759 sys_catalog.cc:263] Verifying existing consensus state
15:27:44.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.109750 13759 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: Bootstrap starting.
15:27:44.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.157160 13759 log.cc:824] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: Log is configured to *not* fsync() on all Append() calls
15:27:44.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.163600 13759 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:27:44.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.164395 13759 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: Bootstrap complete.
15:27:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.209872 13759 raft_consensus.cc:348] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:44.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.213687 13759 raft_consensus.cc:374] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:27:44.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.213989 13759 raft_consensus.cc:729] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e267919edda9441493579257acc2e98b, State: Initialized, Role: FOLLOWER
15:27:44.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.215008 13759 consensus_queue.cc:260] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:44.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.228663 13763 sys_catalog.cc:455] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } } }
15:27:44.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.229771 13763 sys_catalog.cc:458] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [sys.catalog]: This master's current role is: FOLLOWER
15:27:44.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.231243 13759 sys_catalog.cc:564] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [sys.catalog]: configured and running, proceeding with master startup.
15:27:44.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.246212 13713 tablet.cc:2369] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:27:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.272392 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.272732 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.262027 13688 master_runner.cc:186] Error getting master registration for 127.13.3.126:39741: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111),
15:27:44.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.296340 13688 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:27:44.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.467087 13763 raft_consensus.cc:482] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:27:44.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.467666 13763 raft_consensus.cc:504] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:44.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.472635 13708 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111)
15:27:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.477253 13763 leader_election.cc:288] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741), 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307)
15:27:44.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.486096 13709 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307): Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.488121 13708 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741): Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111)
15:27:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.488495 13708 leader_election.cc:302] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e267919edda9441493579257acc2e98b; no voters: 2ae1821a22154b54ba0e6ee844209349, 4eacd7ab7198468f9093b33ce8018ebb
15:27:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.489338 13763 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:27:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.530825 13761 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:27:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.531711 13761 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:27:44.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.532917 13761 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:27:44.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.598997 13761 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:27:44.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.600376 13761 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:27:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.600940 13761 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:27:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.601312 13761 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:27:44.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.601681 13761 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:27:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.602103 13761 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:27:44.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.602360 13761 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46455/jwks.json
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.676610 13761 master_runner.cc:386] Master server non-default flags:
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.3.84:43879
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.3.126:39741,127.13.3.125:35307,127.13.3.124:34635
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/krb5kdc/kudu_127.13.3.126.keytab
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.3.126
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.3.126:39741
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46455/jwks.json
15:27:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/info.pb
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.3.126
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44265
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/logs
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 20 Jun 2026 14:32:01 UTC on ccycloud.quasar-intmyy.root.comops.site
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457507
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:27:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.678201 13761 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:27:44.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:44.679594 13761 file_cache.cc:493] Constructed file cache file cache with capacity 419430
15:27:44.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.706044 13785 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:45.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:45.278883 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:45.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:45.279379 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:46.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.285931 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:46.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.286418 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.618697 13791 raft_consensus.cc:482] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:27:46.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.619724 13791 raft_consensus.cc:504] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:46.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.639773 13791 leader_election.cc:288] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741), 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307)
15:27:46.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.655982 13708 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741): Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111)
15:27:46.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.659211 13709 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307): Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:46.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.659952 13709 leader_election.cc:302] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e267919edda9441493579257acc2e98b; no voters: 2ae1821a22154b54ba0e6ee844209349, 4eacd7ab7198468f9093b33ce8018ebb
15:27:46.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.661046 13791 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:27:46.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:44.707578 13784 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.097630 13783 debug-util.cc:398] Leaking SignalData structure 0x7b0800014920 after lost signal to thread 13761
15:27:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.875108 13761 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.171s user 0.733s sys 1.122s
15:27:46.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.876410 13761 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.172s user 0.734s sys 1.122s
15:27:46.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.881503 13786 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:27:46.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:46.882282 13787 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:27:46.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.882361 13761 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:27:46.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.884078 13761 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:27:46.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.887203 13761 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:27:46.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.888639 13761 hybrid_clock.cc:648] HybridClock initialized: now 1781969266888565 us; error 75 us; skew 500 ppm
15:27:46.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.889806 13761 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:27:46.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:46 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969266, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:46.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.897306 13761 init.cc:377] Logged in from keytab as kudu/127.13.3.126@KRBTEST.COM (short username kudu)
15:27:46.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.902983 13761 webserver.cc:460] Webserver started at http://127.13.3.126:44265/ using document root <none> and password file <none>
15:27:46.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.904142 13761 fs_manager.cc:359] Metadata directory not provided
15:27:46.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.904433 13761 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:27:46.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.915999 13761 fs_manager.cc:709] Time spent opening directory manager: real 0.007s user 0.008s sys 0.000s
15:27:46.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.921149 13800 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:27:46.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.922325 13761 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
15:27:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.922621 13761 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data,/tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2ae1821a22154b54ba0e6ee844209349"
15:27:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-20 15:27:35 on dist-test-slave-fpc8"
15:27:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.924965 13761 fs_report.cc:383] FS layout report
15:27:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:27:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:46.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/data
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:27:46.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:27:46.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.960970 13761 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
15:27:46.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.966743 13761 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:27:46.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:46.967309 13761 kserver.cc:163] Server-wide thread pool size limit: 3276
15:27:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:47.053642 13761 rpc_server.cc:307] RPC server started. Bound to: 127.13.3.126:39741
15:27:47.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:47.053857 13851 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.13.3.126:39741 every 8 connection(s)
15:27:47.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:47.056836 13761 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/data/info.pb
15:27:47.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:47.058686 13325 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu as pid 13761
15:27:47.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:27:47.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:47.086990 13852 sys_catalog.cc:263] Verifying existing consensus state
15:27:47.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260620 15:27:47.089015 13852 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:27:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:47.095616 13761 master.cc:537] Master@127.13.3.126:39741 shutting down...
15:27:47.148 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:47.201824 13761 master.cc:552] Master@127.13.3.126:39741 shutdown complete.
15:27:47.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskNRqU_n/test-tmp/mini-kudu-cluster5869317204274793707/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
15:27:47.237 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.3.124:34635(127.13.3.124:34635)] connecting to peer
15:27:47.247 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.288 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.13.3.124:34635(127.13.3.124:34635)] Successfully connected to peer
15:27:47.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:47.290835 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:47.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:47.291291 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:47.295 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.296 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.305 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.307 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.307 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.308 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.378 [WARN - kudu-nio-1] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:27:47.386 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.13.3.124
15:27:47.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:47 dist-test-slave-fpc8 krb5kdc[13337](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969260, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.3.124@KRBTEST.COM
15:27:47.628 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x195bc415, L:/127.0.0.1:51854 - R:/127.13.3.124:34635] using SASL/GSSAPI
15:27:47.646 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.666 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.670 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.670 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.674 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.675 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.677 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.677 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.682 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.705 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.707 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.710 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.710 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.712 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.713 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.714 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.714 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.725 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.727 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.731 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.731 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.731 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.733 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.744 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.748 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.748 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.748 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.752 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.785 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.787 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.790 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.791 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.791 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.793 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.845 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.847 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.851 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.852 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.853 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.853 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.853 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.925 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.927 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.931 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.931 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.933 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.966 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.968 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.970 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.970 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.972 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.973 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.973 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.973 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:47.985 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:47.986 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:47.988 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.988 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:47.990 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.990 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.991 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:47.992 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:48.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:48.192688 13863 raft_consensus.cc:482] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:27:48.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:48.193107 13863 raft_consensus.cc:504] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:48.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:48.194581 13863 leader_election.cc:288] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741), 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307)
15:27:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:48.200114 13708 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741): Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111)
15:27:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:48.200626 13709 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307): Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:48.200999 13709 leader_election.cc:302] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e267919edda9441493579257acc2e98b; no voters: 2ae1821a22154b54ba0e6ee844209349, 4eacd7ab7198468f9093b33ce8018ebb
15:27:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:48.201656 13863 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:27:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:48.294767 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:48.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:48.295106 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:48.385 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:48.386 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:48.388 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:48.388 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:48.388 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:48.388 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:48.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:48.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:48.390 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:49.297752 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:49.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:49.298045 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:49.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:49.299360 13688 master_runner.cc:186] Error getting master registration for 127.13.3.126:39741: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111),
15:27:49.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:49.299834 13688 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:27:50.045 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:50.047 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:50.048 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:50.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:50.048 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:50.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:50.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:50.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:50.051 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:50.245 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:50.246 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:50.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:50.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:50.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:50.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:50.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:50.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:50.250 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:50.302273 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:50.302608 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:51.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:51.305545 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:51.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:51.305843 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:52.293912 13873 raft_consensus.cc:482] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:27:52.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:52.294322 13873 raft_consensus.cc:504] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:52.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:52.295915 13873 leader_election.cc:288] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741), 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307)
15:27:52.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:52.297781 13708 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111) [suppressed 5 similar messages]
15:27:52.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:52.300372 13709 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307): Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:52.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:52.300827 13708 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741): Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111)
15:27:52.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:52.301191 13708 leader_election.cc:302] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e267919edda9441493579257acc2e98b; no voters: 2ae1821a22154b54ba0e6ee844209349, 4eacd7ab7198468f9093b33ce8018ebb
15:27:52.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:52.301826 13873 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:27:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:52.308688 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:52.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:52.308893 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:53.312753 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:53.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:53.313086 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:53.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:27:53 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969273, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:27:53.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:53.896193 13703 init.cc:295] Successfully reacquired a new kerberos TGT
15:27:54.225 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:54.226 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:54.228 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:54.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:54.228 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:54.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:54.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:54.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:54.231 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:54.303030 13688 master_runner.cc:186] Error getting master registration for 127.13.3.126:39741: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111),
15:27:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:54.303427 13688 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:27:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:54.315881 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:54.316156 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:54.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:54.509776 13883 raft_consensus.cc:482] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:27:54.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:54.510499 13883 raft_consensus.cc:504] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:27:54.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:54.512449 13883 leader_election.cc:288] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741), 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307)
15:27:54.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:54.516165 13708 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741): Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111)
15:27:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:54.516639 13709 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307): Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:27:54.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:54.517033 13709 leader_election.cc:302] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e267919edda9441493579257acc2e98b; no voters: 2ae1821a22154b54ba0e6ee844209349, 4eacd7ab7198468f9093b33ce8018ebb
15:27:54.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:54.517712 13883 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:27:54.565 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:54.566 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:54.568 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:54.568 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:54.568 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:54.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:54.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:54.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:54.570 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:55.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:55.319001 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:55.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:55.319247 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:55.785 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:55.787 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:55.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:55.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:55.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:55.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:55.790 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:55.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:55.792 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:55.914873 13310 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba600 after lost signal to thread 13185
15:27:55.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:55.915704 13310 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba440 after lost signal to thread 13313
15:27:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:56.321906 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:56.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:56.322183 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:56.365 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:27:56.367 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:56.367 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:56.367 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:27:56.367 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:27:56.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:56.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:56.371 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:27:56.372 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:27:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:57.325064 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:57.325344 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:58.328111 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:58.328406 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:27:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:59.306277 13688 master_runner.cc:186] Error getting master registration for 127.13.3.126:39741: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111),
15:27:59.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:27:59.306702 13688 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:27:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:59.331321 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:27:59.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:27:59.331573 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:00.045 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:00.046 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:00.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:00.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:00.050 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:00.050 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:00.050 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:00.051 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:00.053 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:00.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:00.334441 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:00.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:00.334733 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:01.337666 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:01.337953 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:02.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:02.341898 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:02.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:02.342221 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:02.465 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:02.467 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:02.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:02.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:02.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:02.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:02.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:02.470 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:02.471 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:03.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:03.197994 13896 raft_consensus.cc:482] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:28:03.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:03.198565 13896 raft_consensus.cc:504] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:28:03.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:03.200557 13896 leader_election.cc:288] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741), 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307)
15:28:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:03.203233 13708 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111) [suppressed 3 similar messages]
15:28:03.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:03.206271 13709 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307): Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:28:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:03.206861 13708 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741): Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111)
15:28:03.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:03.207242 13708 leader_election.cc:302] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e267919edda9441493579257acc2e98b; no voters: 2ae1821a22154b54ba0e6ee844209349, 4eacd7ab7198468f9093b33ce8018ebb
15:28:03.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:03.208026 13896 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:28:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:03.346645 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:03.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:03.346954 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:03.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:28:03 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969283, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:28:03.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:03.898393 13703 init.cc:295] Successfully reacquired a new kerberos TGT
15:28:04.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:04.309238 13688 master_runner.cc:186] Error getting master registration for 127.13.3.126:39741: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111),
15:28:04.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:04.309610 13688 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:28:04.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:04.349663 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:04.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:04.349916 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:05.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:05.352813 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:05.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:05.353065 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:06.145 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:06.147 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:06.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:06.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:06.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:06.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:06.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:06.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:06.151 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:06.355911 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:06.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:06.356163 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:07.359090 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:07.359448 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:07.685 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:07.686 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:07.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:07.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:07.688 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:07.688 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:07.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:07.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:07.690 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:07.745 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:07.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:07.747 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:07.747 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:07.747 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:07.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:07.748 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:07.748 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:07.751 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:08.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:08.362433 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:08.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:08.362700 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:09.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:09.312279 13688 master_runner.cc:186] Error getting master registration for 127.13.3.126:39741: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111),
15:28:09.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:09.312666 13688 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:28:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:09.365381 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:09.365649 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:09.745 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:09.746 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:09.747 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:09.747 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:09.747 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:09.747 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:09.747 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:09.747 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:09.749 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:10.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:10.368424 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:10.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:10.368744 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:10.678970 13908 raft_consensus.cc:482] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:28:10.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:10.679363 13908 raft_consensus.cc:504] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "2ae1821a22154b54ba0e6ee844209349" member_type: VOTER last_known_addr { host: "127.13.3.126" port: 39741 } } peers { permanent_uuid: "4eacd7ab7198468f9093b33ce8018ebb" member_type: VOTER last_known_addr { host: "127.13.3.125" port: 35307 } } peers { permanent_uuid: "e267919edda9441493579257acc2e98b" member_type: VOTER last_known_addr { host: "127.13.3.124" port: 34635 } }
15:28:10.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:10.680842 13908 leader_election.cc:288] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741), 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307)
15:28:10.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:10.682945 13708 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111) [suppressed 1 similar messages]
15:28:10.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:10.685603 13709 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4eacd7ab7198468f9093b33ce8018ebb (127.13.3.125:35307): Network error: Client connection negotiation failed: client connection to 127.13.3.125:35307: connect: Connection refused (error 111)
15:28:10.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:10.686074 13708 leader_election.cc:334] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 2ae1821a22154b54ba0e6ee844209349 (127.13.3.126:39741): Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111)
15:28:10.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:10.686403 13708 leader_election.cc:302] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: e267919edda9441493579257acc2e98b; no voters: 2ae1821a22154b54ba0e6ee844209349, 4eacd7ab7198468f9093b33ce8018ebb
15:28:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:10.687041 13908 raft_consensus.cc:2726] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
15:28:11.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:11.371631 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:11.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:11.371901 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:11.945 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:11.946 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:11.947 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:11.947 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:11.947 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:11.947 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:11.947 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:11.947 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:11.949 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:12.374528 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:12.374781 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:13.225 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:13.226 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:13.227 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:13.227 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:13.227 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:13.228 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:13.228 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:13.228 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:13.230 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:13.377908 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:13.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:13.378234 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 20 15:28:13 dist-test-slave-fpc8 krb5kdc[13337](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781969293, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.3.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:28:13.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:13.900638 13703 init.cc:295] Successfully reacquired a new kerberos TGT
15:28:14.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:14.316159 13688 master_runner.cc:186] Error getting master registration for 127.13.3.126:39741: Network error: Client connection negotiation failed: client connection to 127.13.3.126:39741: connect: Connection refused (error 111),
15:28:14.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:14.316587 13688 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:28:14.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:14.380780 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:14.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:14.381023 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:15.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:15.383566 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:15.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:15.383806 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:16.125 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:16.126 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:16.127 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:16.127 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:16.128 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:16.128 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:16.128 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:16.128 [INFO - pool-6-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:16.130 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:16.386129 13774 catalog_manager.cc:1558] T 00000000000000000000000000000000 P e267919edda9441493579257acc2e98b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:28:16.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260620 15:28:16.386371 13774 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:28:16.805 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.125:35307(127.13.3.125:35307)] connecting to peer
15:28:16.806 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.13.3.126:39741(127.13.3.126:39741)] connecting to peer
15:28:16.807 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:16.807 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:16.807 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.125:35307(127.13.3.125:35307)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:16.807 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.3.126:39741(127.13.3.126:39741)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:16.808 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.125:35307: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307
15:28:16.808 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.3.126:39741: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
15:28:16.810 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741; will retry
15:28:16.814 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29746), Traces: [0ms] refreshing cache from master, [69ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [169ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [229ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [239ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [573ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [602ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [607ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [613ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [614ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [642ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [644ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [645ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [647ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [661ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [663ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [664ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [680ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [682ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [684ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [721ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [723ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [724ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [725ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [782ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [784ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [785ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [861ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [862ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [864ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [902ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [904ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [905ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [906ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [920ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [922ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [923ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [1317ms] refreshing cache from master, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [1322ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [2977ms] refreshing cache from master, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [2982ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [2983ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [3176ms] refreshing cache from master, [3177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [3178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [3178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [3180ms] too many traces: truncated at 100 traces, deferred=Deferred@357015590(state=PENDING, result=null, callback=(continuation of Deferred@628590925 after retry RPC after error@645308854) -> (continuation of Deferred@2021946405 after retry RPC after error@1349172611) -> (continuation of Deferred@1229876002 after retry RPC after error@1520830636) -> (continuation of Deferred@22571649 after retry RPC after error@81480076) -> (continuation of Deferred@1056318798 after retry RPC after error@1426041172) -> (continuation of Deferred@372366632 after retry RPC after error@1215848779) -> (continuation of Deferred@1194062175 after retry RPC after error@1857504743) -> (continuation of Deferred@1995902340 after retry RPC after error@2106480680) -> (continuation of Deferred@1698164687 after retry RPC after error@830150387) -> (continuation of Deferred@2142152309 after retry RPC after error@1886322352) -> (continuation of Deferred@712851173 after retry RPC after error@1129867735) -> (continuation of Deferred@1238364871 after retry RPC after error@1322458672) -> (continuation of Deferred@219847449 after retry RPC after error@227298748) -> (continuation of Deferred@1004789504 after retry RPC after error@1838638290) -> (continuation of Deferred@873265194 after retry RPC after error@1535292090) -> (continuation of Deferred@2126282005 after retry RPC after error@685455332) -> (continuation of Deferred@2068137853 after retry RPC after error@891039181) -> (continuation of Deferred@685095598 after retry RPC after error@1505807431) -> (continuation of Deferred@141498029 after retry RPC after error@1548300328) -> (continuation of Deferred@1506599365 after retry RPC after error@103922767) -> (continuation of Deferred@1981487194 after retry RPC after error@1665580737) -> (continuation of Deferred@1572518309 after retry RPC after error@1457694105) -> (continuation of Deferred@2014686935 after retry RPC after error@787724194) -> (continuation of Deferred@638338120 after retry RPC after error@1694622589) -> (continuation of Deferred@808946204 after retry RPC after error@55211295) -> (continuation of Deferred@1193299770 after retry RPC after error@768624296), errback=(continuation of Deferred@628590925 after retry RPC after error@645308854) -> (continuation of Deferred@2021946405 after retry RPC after error@1349172611) -> (continuation of Deferred@1229876002 after retry RPC after error@1520830636) -> (continuation of Deferred@22571649 after retry RPC after error@81480076) -> (continuation of Deferred@1056318798 after retry RPC after error@1426041172) -> (continuation of Deferred@372366632 after retry RPC after error@1215848779) -> (continuation of Deferred@1194062175 after retry RPC after error@1857504743) -> (continuation of Deferred@1995902340 after retry RPC after error@2106480680) -> (continuation of Deferred@1698164687 after retry RPC after error@830150387) -> (continuation of Deferred@2142152309 after retry RPC after error@1886322352) -> (continuation of Deferred@712851173 after retry RPC after error@1129867735) -> (continuation of Deferred@1238364871 after retry RPC after error@1322458672) -> (continuation of Deferred@219847449 after retry RPC after error@227298748) -> (continuation of Deferred@1004789504 after retry RPC after error@1838638290) -> (continuation of Deferred@873265194 after retry RPC after error@1535292090) -> (continuation of Deferred@2126282005 after retry RPC after error@685455332) -> (continuation of Deferred@2068137853 after retry RPC after error@891039181) -> (continuation of Deferred@685095598 after retry RPC after error@1505807431) -> (continuation of Deferred@141498029 after retry RPC after error@1548300328) -> (continuation of Deferred@1506599365 after retry RPC after error@103922767) -> (continuation of Deferred@1981487194 after retry RPC after error@1665580737) -> (continuation of Deferred@1572518309 after retry RPC after error@1457694105) -> (continuation of Deferred@2014686935 after retry RPC after error@787724194) -> (continuation of Deferred@638338120 after retry RPC after error@1694622589) -> (continuation of Deferred@808946204 after retry RPC after error@55211295) -> (continuation of Deferred@1193299770 after retry RPC after error@768624296)))
15:28:16.820 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.3.125:35307(127.13.3.125:35307)] disconnecting while in state TERMINATED
15:28:16.821 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.3.126:39741(127.13.3.126:39741)] disconnecting while in state TERMINATED
15:28:16.821 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.3.124:34635(127.13.3.124:34635)] disconnecting while in state READY
15:28:16.826 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:28:16.832 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:28:16.832 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.13.3.124:34635(127.13.3.124:34635)] handling channelInactive
15:28:16.833 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.3.124:34635(127.13.3.124:34635)] cleaning up while in state READY due to: connection closed
15:28:16.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:16.840179 13325 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 13688
15:28:18.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:28:18Z chronyd exiting
15:28:19.053 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:28:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:19.055146 12597 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 12877
15:28:19.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:19.083081 12597 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 13037
15:28:19.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:19.109495 12597 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 13182
15:28:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:19.138834 12597 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 12609
15:28:19.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:19.170708 12597 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 12679
15:28:19.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260620 15:28:19.202260 12597 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskNRqU_n/build/tsan/bin/kudu with pid 12754
15:28:20.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-20T15:28:20Z chronyd exiting
15:28:21.395 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29746), Traces: [0ms] refreshing cache from master, [69ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [169ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [229ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [239ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [573ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [602ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [607ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [613ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [614ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [642ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [644ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [645ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [647ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [661ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [663ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [664ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [680ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [682ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [684ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [721ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [723ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [724ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [725ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [782ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [784ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [785ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [861ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [862ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [864ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [902ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [904ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [905ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [906ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [920ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [922ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [923ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [1317ms] refreshing cache from master, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [1322ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [2977ms] refreshing cache from master, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [2982ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [2983ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [3176ms] refreshing cache from master, [3177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [3178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [3178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [3180ms] too many traces: truncated at 100 traces, deferred=Deferred@357015590(state=PENDING, result=null, callback=(continuation of Deferred@628590925 after retry RPC after error@645308854) -> (continuation of Deferred@2021946405 after retry RPC after error@1349172611) -> (continuation of Deferred@1229876002 after retry RPC after error@1520830636) -> (continuation of Deferred@22571649 after retry RPC after error@81480076) -> (continuation of Deferred@1056318798 after retry RPC after error@1426041172) -> (continuation of Deferred@372366632 after retry RPC after error@1215848779) -> (continuation of Deferred@1194062175 after retry RPC after error@1857504743) -> (continuation of Deferred@1995902340 after retry RPC after error@2106480680) -> (continuation of Deferred@1698164687 after retry RPC after error@830150387) -> (continuation of Deferred@2142152309 after retry RPC after error@1886322352) -> (continuation of Deferred@712851173 after retry RPC after error@1129867735) -> (continuation of Deferred@1238364871 after retry RPC after error@1322458672) -> (continuation of Deferred@219847449 after retry RPC after error@227298748) -> (continuation of Deferred@1004789504 after retry RPC after error@1838638290) -> (continuation of Deferred@873265194 after retry RPC after error@1535292090) -> (continuation of Deferred@2126282005 after retry RPC after error@685455332) -> (continuation of Deferred@2068137853 after retry RPC after error@891039181) -> (continuation of Deferred@685095598 after retry RPC after error@1505807431) -> (continuation of Deferred@141498029 after retry RPC after error@1548300328) -> (continuation of Deferred@1506599365 after retry RPC after error@103922767) -> (continuation of Deferred@1981487194 after retry RPC after error@1665580737) -> (continuation of Deferred@1572518309 after retry RPC after error@1457694105) -> (continuation of Deferred@2014686935 after retry RPC after error@787724194) -> (continuation of Deferred@638338120 after retry RPC after error@1694622589) -> (continuation of Deferred@808946204 after retry RPC after error@55211295) -> (continuation of Deferred@1193299770 after retry RPC after error@768624296), errback=(continuation of Deferred@628590925 after retry RPC after error@645308854) -> (continuation of Deferred@2021946405 after retry RPC after error@1349172611) -> (continuation of Deferred@1229876002 after retry RPC after error@1520830636) -> (continuation of Deferred@22571649 after retry RPC after error@81480076) -> (continuation of Deferred@1056318798 after retry RPC after error@1426041172) -> (continuation of Deferred@372366632 after retry RPC after error@1215848779) -> (continuation of Deferred@1194062175 after retry RPC after error@1857504743) -> (continuation of Deferred@1995902340 after retry RPC after error@2106480680) -> (continuation of Deferred@1698164687 after retry RPC after error@830150387) -> (continuation of Deferred@2142152309 after retry RPC after error@1886322352) -> (continuation of Deferred@712851173 after retry RPC after error@1129867735) -> (continuation of Deferred@1238364871 after retry RPC after error@1322458672) -> (continuation of Deferred@219847449 after retry RPC after error@227298748) -> (continuation of Deferred@1004789504 after retry RPC after error@1838638290) -> (continuation of Deferred@873265194 after retry RPC after error@1535292090) -> (continuation of Deferred@2126282005 after retry RPC after error@685455332) -> (continuation of Deferred@2068137853 after retry RPC after error@891039181) -> (continuation of Deferred@685095598 after retry RPC after error@1505807431) -> (continuation of Deferred@141498029 after retry RPC after error@1548300328) -> (continuation of Deferred@1506599365 after retry RPC after error@103922767) -> (continuation of Deferred@1981487194 after retry RPC after error@1665580737) -> (continuation of Deferred@1572518309 after retry RPC after error@1457694105) -> (continuation of Deferred@2014686935 after retry RPC after error@787724194) -> (continuation of Deferred@638338120 after retry RPC after error@1694622589) -> (continuation of Deferred@808946204 after retry RPC after error@55211295) -> (continuation of Deferred@1193299770 after retry RPC after error@768624296)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.13.3.126:39741
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=27, TimeoutTracker(timeout=30000, elapsed=29746), Traces: [0ms] refreshing cache from master, [69ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [169ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [229ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [239ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [573ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [578ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [597ms] refreshing cache from master, [598ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [600ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [602ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [602ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [607ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [613ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [614ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [637ms] refreshing cache from master, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [639ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [642ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [644ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [645ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [647ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [657ms] refreshing cache from master, [657ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [658ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [661ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [663ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [664ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [665ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [676ms] refreshing cache from master, [676ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [680ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [682ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [684ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [684ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [716ms] refreshing cache from master, [717ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [718ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [721ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [723ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [724ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [725ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [776ms] refreshing cache from master, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [779ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [782ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [784ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [785ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [785ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [856ms] refreshing cache from master, [856ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [858ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [859ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [861ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [862ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [864ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [865ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [897ms] refreshing cache from master, [897ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [898ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [900ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [902ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [904ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [905ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [906ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [917ms] refreshing cache from master, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [918ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [920ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [922ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [923ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [924ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [1317ms] refreshing cache from master, [1317ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [1318ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [1321ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [1322ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [1322ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [2977ms] refreshing cache from master, [2977ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [2978ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [2979ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.126:39741: Network error: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741, [2981ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.125:35307: Network error: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [2982ms] Sub RPC ConnectToMaster: received response from server master-127.13.3.124:34635: OK, [2983ms] delaying RPC due to: Service unavailable: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307, [3176ms] refreshing cache from master, [3177ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.125:35307, [3178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.124:34635, [3178ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.3.126:39741, [3180ms] too many traces: truncated at 100 traces, deferred=Deferred@357015590(state=PENDING, result=null, callback=(continuation of Deferred@628590925 after retry RPC after error@645308854) -> (continuation of Deferred@2021946405 after retry RPC after error@1349172611) -> (continuation of Deferred@1229876002 after retry RPC after error@1520830636) -> (continuation of Deferred@22571649 after retry RPC after error@81480076) -> (continuation of Deferred@1056318798 after retry RPC after error@1426041172) -> (continuation of Deferred@372366632 after retry RPC after error@1215848779) -> (continuation of Deferred@1194062175 after retry RPC after error@1857504743) -> (continuation of Deferred@1995902340 after retry RPC after error@2106480680) -> (continuation of Deferred@1698164687 after retry RPC after error@830150387) -> (continuation of Deferred@2142152309 after retry RPC after error@1886322352) -> (continuation of Deferred@712851173 after retry RPC after error@1129867735) -> (continuation of Deferred@1238364871 after retry RPC after error@1322458672) -> (continuation of Deferred@219847449 after retry RPC after error@227298748) -> (continuation of Deferred@1004789504 after retry RPC after error@1838638290) -> (continuation of Deferred@873265194 after retry RPC after error@1535292090) -> (continuation of Deferred@2126282005 after retry RPC after error@685455332) -> (continuation of Deferred@2068137853 after retry RPC after error@891039181) -> (continuation of Deferred@685095598 after retry RPC after error@1505807431) -> (continuation of Deferred@141498029 after retry RPC after error@1548300328) -> (continuation of Deferred@1506599365 after retry RPC after error@103922767) -> (continuation of Deferred@1981487194 after retry RPC after error@1665580737) -> (continuation of Deferred@1572518309 after retry RPC after error@1457694105) -> (continuation of Deferred@2014686935 after retry RPC after error@787724194) -> (continuation of Deferred@638338120 after retry RPC after error@1694622589) -> (continuation of Deferred@808946204 after retry RPC after error@55211295) -> (continuation of Deferred@1193299770 after retry RPC after error@768624296), errback=(continuation of Deferred@628590925 after retry RPC after error@645308854) -> (continuation of Deferred@2021946405 after retry RPC after error@1349172611) -> (continuation of Deferred@1229876002 after retry RPC after error@1520830636) -> (continuation of Deferred@22571649 after retry RPC after error@81480076) -> (continuation of Deferred@1056318798 after retry RPC after error@1426041172) -> (continuation of Deferred@372366632 after retry RPC after error@1215848779) -> (continuation of Deferred@1194062175 after retry RPC after error@1857504743) -> (continuation of Deferred@1995902340 after retry RPC after error@2106480680) -> (continuation of Deferred@1698164687 after retry RPC after error@830150387) -> (continuation of Deferred@2142152309 after retry RPC after error@1886322352) -> (continuation of Deferred@712851173 after retry RPC after error@1129867735) -> (continuation of Deferred@1238364871 after retry RPC after error@1322458672) -> (continuation of Deferred@219847449 after retry RPC after error@227298748) -> (continuation of Deferred@1004789504 after retry RPC after error@1838638290) -> (continuation of Deferred@873265194 after retry RPC after error@1535292090) -> (continuation of Deferred@2126282005 after retry RPC after error@685455332) -> (continuation of Deferred@2068137853 after retry RPC after error@891039181) -> (continuation of Deferred@685095598 after retry RPC after error@1505807431) -> (continuation of Deferred@141498029 after retry RPC after error@1548300328) -> (continuation of Deferred@1506599365 after retry RPC after error@103922767) -> (continuation of Deferred@1981487194 after retry RPC after error@1665580737) -> (continuation of Deferred@1572518309 after retry RPC after error@1457694105) -> (continuation of Deferred@2014686935 after retry RPC after error@787724194) -> (continuation of Deferred@638338120 after retry RPC after error@1694622589) -> (continuation of Deferred@808946204 after retry RPC after error@55211295) -> (continuation of Deferred@1193299770 after retry RPC after error@768624296)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.13.3.125:35307,127.13.3.124:34635,127.13.3.126:39741) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.125:35307(127.13.3.125:35307): Connection refused: /127.13.3.125:35307,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.13.3.126:39741(127.13.3.126:39741): Connection refused: /127.13.3.126:39741
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.13.3.126:39741
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)