20:07:41.246 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
20:07:41.246 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_xrIj3/test-tmp
20:07:41.247 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:07:41.247 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_xrIj3/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:07:41.250 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
20:07:41.638 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:07:41.638 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:07:41.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:41.642843 16458 mini_oidc.cc:150] Starting JWKS server
20:07:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:41.647048 16458 webserver.cc:533] Webserver started at http://127.0.0.1:32831/ using document root <none> and password file <none>
20:07:41.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:41.647523 16458 mini_oidc.cc:186] Starting OIDC Discovery server
20:07:41.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:41.649422 16458 webserver.cc:533] Webserver started at http://127.0.0.1:42199/ using document root <none> and password file <none>
20:07:41.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T20:07:41Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:07:41.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T20:07:41Z Disabled control of system clock
20:07:41.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:41.720463 16458 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/wal
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/data
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/logs
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/data/info.pb
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.190:42753
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.190
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:41.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32831/jwks.json with env {}
20:07:42.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.070335 16470 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:07:42.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.071096 16470 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:42.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.071599 16470 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:07:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.108778 16470 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:07:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.109135 16470 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:07:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.109336 16470 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.109556 16470 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:07:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.109750 16470 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:42.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.109910 16470 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32831/jwks.json
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:42.154426 16470 master_runner.cc:387] Master server non-default flags:
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/data
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/wal
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.190:42753
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32831/jwks.json
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/data/info.pb
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.190
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/logs
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:07:42.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:42.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:42.156337 16470 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:42.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:42.158289 16470 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.171566 16476 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:43.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.173362 16477 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:43.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:42.178633 16479 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.438792 16478 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:07:43.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.438796 16470 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:43.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.444026 16470 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:43.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.447227 16470 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:43.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.448683 16470 hybrid_clock.cc:648] HybridClock initialized: now 1781640463448609 us; error 92 us; skew 500 ppm
20:07:43.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.449635 16470 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:43.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.456389 16470 webserver.cc:533] Webserver started at http://127.16.18.190:42783/ using document root <none> and password file <none>
20:07:43.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.457442 16470 fs_manager.cc:362] Metadata directory not provided
20:07:43.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.457690 16470 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:43.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.458199 16470 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:07:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.462754 16470 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/data/instance:
20:07:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da179f21744a413ca9ee8255d492a8d3"
20:07:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:43 on dist-test-slave-mfps"
20:07:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.464021 16470 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/wal/instance:
20:07:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da179f21744a413ca9ee8255d492a8d3"
20:07:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:43 on dist-test-slave-mfps"
20:07:43.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.471841 16470 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
20:07:43.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.478531 16486 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.480113 16470 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
20:07:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.480516 16470 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/wal
20:07:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "da179f21744a413ca9ee8255d492a8d3"
20:07:43.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:43 on dist-test-slave-mfps"
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.480878 16470 fs_report.cc:389] FS layout report
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/wal
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/wal
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/data/data
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:43.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:43.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.530958 16470 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:43.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.532603 16470 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:43.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.533108 16470 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:43.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.612135 16470 rpc_server.cc:307] RPC server started. Bound to: 127.16.18.190:42753
20:07:43.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.612289 16537 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.18.190:42753 every 8 connection(s)
20:07:43.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.615129 16470 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/data/info.pb
20:07:43.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.621973 16458 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 16470
20:07:43.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.622699 16458 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-0/wal/instance
20:07:43.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.624184 16538 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.629249 16458 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/wal
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/data
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:43.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/logs
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/data/info.pb
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.189:45015
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.189
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:43.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32831/jwks.json with env {}
20:07:43.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.642731 16538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } has no permanent_uuid. Determining permanent_uuid...
20:07:43.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.678125 16538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } has no permanent_uuid. Determining permanent_uuid...
20:07:43.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.680856 16488 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.18.189:45015: connect: Connection refused (error 111)
20:07:43.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.686028 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.189:45015: Network error: Client connection negotiation failed: client connection to 127.16.18.189:45015: connect: Connection refused (error 111)
20:07:43.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.735752 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } attempt: 1
20:07:43.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.740934 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.189:45015: Network error: Client connection negotiation failed: client connection to 127.16.18.189:45015: connect: Connection refused (error 111)
20:07:43.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.809470 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } attempt: 2
20:07:43.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.814505 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.189:45015: Network error: Client connection negotiation failed: client connection to 127.16.18.189:45015: connect: Connection refused (error 111)
20:07:43.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:43.906061 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } attempt: 3
20:07:43.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.911087 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.189:45015: Network error: Client connection negotiation failed: client connection to 127.16.18.189:45015: connect: Connection refused (error 111)
20:07:43.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.959964 16540 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:07:43.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.960516 16540 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:43.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.960976 16540 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:07:43.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.994853 16540 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:07:43.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.995151 16540 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:07:43.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.995350 16540 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:43.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.995556 16540 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:07:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.995836 16540 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:43.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:43.995999 16540 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32831/jwks.json
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:44.033855 16540 master_runner.cc:387] Master server non-default flags:
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/data
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/wal
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.189:45015
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32831/jwks.json
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/data/info.pb
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.189
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:44.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/logs
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:44.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:44.035207 16540 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:44.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:44.037001 16540 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:44.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:44.049371 16549 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:44.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:44.054590 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } attempt: 4
20:07:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:44.062084 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.189:45015: Network error: Client connection negotiation failed: client connection to 127.16.18.189:45015: connect: Connection refused (error 111)
20:07:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:44.361722 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } attempt: 5
20:07:44.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:44.369557 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.189:45015: Network error: Client connection negotiation failed: client connection to 127.16.18.189:45015: connect: Connection refused (error 111)
20:07:44.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:44.917248 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } attempt: 6
20:07:44.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:44.930883 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.189:45015: Network error: Client connection negotiation failed: client connection to 127.16.18.189:45015: connect: Connection refused (error 111)
20:07:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:44.050813 16550 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:45.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:44.054087 16552 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:45.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.308090 16551 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1251 milliseconds
20:07:45.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.308264 16540 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:45.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.309927 16540 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:45.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.313686 16540 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.315238 16540 hybrid_clock.cc:648] HybridClock initialized: now 1781640465315166 us; error 97 us; skew 500 ppm
20:07:45.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.316260 16540 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:45.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.323835 16540 webserver.cc:533] Webserver started at http://127.16.18.189:46849/ using document root <none> and password file <none>
20:07:45.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.324934 16540 fs_manager.cc:362] Metadata directory not provided
20:07:45.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.325158 16540 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:45.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.325629 16540 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:07:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.330708 16540 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/data/instance:
20:07:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "348c7a4769db4e028451bbe76fa0ab73"
20:07:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:45 on dist-test-slave-mfps"
20:07:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.332046 16540 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/wal/instance:
20:07:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "348c7a4769db4e028451bbe76fa0ab73"
20:07:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:45 on dist-test-slave-mfps"
20:07:45.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.340708 16540 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.001s
20:07:45.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.347554 16561 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.349046 16540 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
20:07:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.349440 16540 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/wal
20:07:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "348c7a4769db4e028451bbe76fa0ab73"
20:07:45.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:45 on dist-test-slave-mfps"
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.349823 16540 fs_report.cc:389] FS layout report
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/wal
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/wal
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/data/data
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:45.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:45.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.409946 16540 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.411789 16540 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.412261 16540 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:45.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.500340 16540 rpc_server.cc:307] RPC server started. Bound to: 127.16.18.189:45015
20:07:45.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.500409 16612 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.18.189:45015 every 8 connection(s)
20:07:45.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.503548 16540 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/data/info.pb
20:07:45.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.509378 16458 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 16540
20:07:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.509898 16458 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-1/wal/instance
20:07:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.511127 16613 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.512429 16458 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/wal
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/data
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:45.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/logs
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/data/info.pb
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.188:36443
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.188
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32831/jwks.json with env {}
20:07:45.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.525861 16613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } has no permanent_uuid. Determining permanent_uuid...
20:07:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.555902 16613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } has no permanent_uuid. Determining permanent_uuid...
20:07:45.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.573532 16613 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } has no permanent_uuid. Determining permanent_uuid...
20:07:45.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.576639 16562 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:45.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.582140 16613 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:45.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.631769 16613 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 1
20:07:45.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.637259 16613 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:45.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.705750 16613 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 2
20:07:45.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.711428 16613 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:45.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.802989 16613 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 3
20:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.809434 16613 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.844771 16615 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:07:45.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.845532 16615 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:45.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.846046 16615 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:07:45.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.881258 16615 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:07:45.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.881640 16615 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:07:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.881858 16615 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.882068 16615 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:07:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.882261 16615 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:45.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.882417 16615 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:32831/jwks.json
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.923972 16615 master_runner.cc:387] Master server non-default flags:
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/data
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/wal
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.188:36443
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:32831/jwks.json
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/data/info.pb
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.188
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/logs
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:07:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:07:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:07:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.925765 16615 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.927541 16615 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:45.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.940174 16625 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:45.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.952961 16613 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 4
20:07:45.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.957320 16613 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:45.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:45.991608 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } attempt: 7
20:07:46.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:46.014793 16538 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } has no permanent_uuid. Determining permanent_uuid...
20:07:46.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:46.026067 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:46.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:46.084766 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 1
20:07:46.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:46.093616 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:46.175319 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 2
20:07:46.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:46.183619 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:46.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:46.256811 16613 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 5
20:07:46.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:46.265686 16613 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:46.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:46.269282 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 3
20:07:46.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:46.276964 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:46.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:46.417631 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 4
20:07:46.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:46.425776 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:46.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:46.709514 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 5
20:07:46.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:46.718372 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:46.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:46.813407 16613 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 6
20:07:46.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:46.826488 16613 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:47.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:45.941783 16626 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:47.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:47.231302 16628 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:47.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:47.234654 16627 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1289 milliseconds
20:07:47.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.234836 16615 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:47.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.236671 16615 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:47.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.240319 16615 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:47.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.241981 16615 hybrid_clock.cc:648] HybridClock initialized: now 1781640467241890 us; error 108 us; skew 500 ppm
20:07:47.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.243027 16615 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:47.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.251004 16615 webserver.cc:533] Webserver started at http://127.16.18.188:40579/ using document root <none> and password file <none>
20:07:47.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.252185 16615 fs_manager.cc:362] Metadata directory not provided
20:07:47.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.252461 16615 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.253062 16615 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:07:47.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.258366 16615 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/data/instance:
20:07:47.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23fb1b979706471688cc541e571209ba"
20:07:47.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:47 on dist-test-slave-mfps"
20:07:47.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.259716 16615 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/wal/instance:
20:07:47.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23fb1b979706471688cc541e571209ba"
20:07:47.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:47 on dist-test-slave-mfps"
20:07:47.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.268571 16615 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.003s sys 0.004s
20:07:47.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.271143 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 6
20:07:47.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.276644 16640 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:47.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.278169 16615 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.002s
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.278592 16615 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/wal
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "23fb1b979706471688cc541e571209ba"
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:47 on dist-test-slave-mfps"
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:47.279265 16538 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.18.188:36443: Network error: Client connection negotiation failed: client connection to 127.16.18.188:36443: connect: Connection refused (error 111)
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.279026 16615 fs_report.cc:389] FS layout report
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/wal
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/wal
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/data/data
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:47.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:47.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.335110 16615 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:47.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.336709 16615 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:47.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.337155 16615 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:47.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.418282 16615 rpc_server.cc:307] RPC server started. Bound to: 127.16.18.188:36443
20:07:47.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.418428 16692 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.18.188:36443 every 8 connection(s)
20:07:47.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.421149 16615 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/data/info.pb
20:07:47.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.427387 16458 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 16615
20:07:47.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.428047 16458 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/master-2/wal/instance
20:07:47.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.429735 16693 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:07:47.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.452486 16693 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } has no permanent_uuid. Determining permanent_uuid...
20:07:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.485157 16693 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } has no permanent_uuid. Determining permanent_uuid...
20:07:47.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.497802 16693 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } has no permanent_uuid. Determining permanent_uuid...
20:07:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.519117 16693 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba: Bootstrap starting.
20:07:47.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.528084 16693 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba: Neither blocks nor log segments found. Creating new log.
20:07:47.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.531124 16693 log.cc:826] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba: Log is configured to *not* fsync() on all Append() calls
20:07:47.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.538165 16693 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba: No bootstrap required, opened a new log
20:07:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.559197 16693 raft_consensus.cc:359] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.559923 16693 raft_consensus.cc:385] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:47.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.560115 16693 raft_consensus.cc:740] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23fb1b979706471688cc541e571209ba, State: Initialized, Role: FOLLOWER
20:07:47.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.560817 16693 consensus_queue.cc:260] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.563442 16702 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:47.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.564201 16702 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: This master's current role is: FOLLOWER
20:07:47.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.564795 16693 sys_catalog.cc:565] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: configured and running, proceeding with master startup.
20:07:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:47.592339 16713 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:07:47.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:47.592744 16713 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:07:47.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:47.593194 16647 tablet.cc:2406] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:07:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.586831 16615 master_runner.cc:187] Error getting master registration for 127.16.18.190:42753: OK, instance_id { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" instance_seqno: 1781640463589735 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:07:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.618065 16615 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:07:47.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.887166 16613 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 7
20:07:47.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.911572 16613 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: Bootstrap starting.
20:07:47.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.920429 16613 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: Neither blocks nor log segments found. Creating new log.
20:07:47.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.922847 16613 log.cc:826] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: Log is configured to *not* fsync() on all Append() calls
20:07:47.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.930271 16613 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: No bootstrap required, opened a new log
20:07:47.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.952605 16613 raft_consensus.cc:359] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:47.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.953416 16613 raft_consensus.cc:385] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:47.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.953658 16613 raft_consensus.cc:740] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 348c7a4769db4e028451bbe76fa0ab73, State: Initialized, Role: FOLLOWER
20:07:47.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.954723 16613 consensus_queue.cc:260] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:47.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.959162 16613 sys_catalog.cc:565] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [sys.catalog]: configured and running, proceeding with master startup.
20:07:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.957592 16717 sys_catalog.cc:455] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.960669 16717 sys_catalog.cc:458] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [sys.catalog]: This master's current role is: FOLLOWER
20:07:48.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:48.007427 16728 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:07:48.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:48.007916 16728 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:07:48.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:47.991057 16540 master_runner.cc:187] Error getting master registration for 127.16.18.190:42753: OK, instance_id { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" instance_seqno: 1781640463589735 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:07:48.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.022370 16540 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:07:48.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.051901 16702 raft_consensus.cc:493] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:07:48.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.052523 16702 raft_consensus.cc:515] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:48.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.056861 16588 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23fb1b979706471688cc541e571209ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "348c7a4769db4e028451bbe76fa0ab73" is_pre_election: true
20:07:48.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.057216 16702 leader_election.cc:290] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers da179f21744a413ca9ee8255d492a8d3 (127.16.18.190:42753), 348c7a4769db4e028451bbe76fa0ab73 (127.16.18.189:45015)
20:07:48.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.057798 16588 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 23fb1b979706471688cc541e571209ba in term 0.
20:07:48.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.059494 16643 leader_election.cc:304] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23fb1b979706471688cc541e571209ba, 348c7a4769db4e028451bbe76fa0ab73; no voters:
20:07:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.060716 16702 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 0 FOLLOWER]: Leader pre-election won for term 1
20:07:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.061142 16702 raft_consensus.cc:493] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
20:07:48.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.061547 16702 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 0 FOLLOWER]: Advancing to term 1
20:07:48.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.071118 16702 raft_consensus.cc:515] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:48.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.085438 16588 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23fb1b979706471688cc541e571209ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "348c7a4769db4e028451bbe76fa0ab73"
20:07:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.086201 16588 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [term 0 FOLLOWER]: Advancing to term 1
20:07:48.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.096174 16588 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 23fb1b979706471688cc541e571209ba in term 1.
20:07:48.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.096907 16702 leader_election.cc:290] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [CANDIDATE]: Term 1 election: Requested vote from peers da179f21744a413ca9ee8255d492a8d3 (127.16.18.190:42753), 348c7a4769db4e028451bbe76fa0ab73 (127.16.18.189:45015)
20:07:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.098645 16643 leader_election.cc:304] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 23fb1b979706471688cc541e571209ba, 348c7a4769db4e028451bbe76fa0ab73; no voters:
20:07:48.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.099602 16702 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 1 FOLLOWER]: Leader election won for term 1
20:07:48.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.101998 16702 raft_consensus.cc:697] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [term 1 LEADER]: Becoming Leader. State: Replica: 23fb1b979706471688cc541e571209ba, State: Running, Role: LEADER
20:07:48.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.103576 16702 consensus_queue.cc:237] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.124614 16730 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23fb1b979706471688cc541e571209ba. Latest consensus state: current_term: 1 leader_uuid: "23fb1b979706471688cc541e571209ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.125550 16730 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: This master's current role is: LEADER
20:07:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.129861 16732 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
20:07:48.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.139110 16732 catalog_manager.cc:1519] Initializing Kudu cluster ID...
20:07:48.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.160208 16588 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [term 1 FOLLOWER]: Refusing update from remote peer 23fb1b979706471688cc541e571209ba: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:07:48.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.165295 16730 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [LEADER]: Connected to new peer: Peer: permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:07:48.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:48.171665 16567 tablet.cc:2406] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:07:48.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.197994 16733 mvcc.cc:204] Tried to move back new op lower bound from 7297599357563482112 to 7297599357377409024. Current Snapshot: MvccSnapshot[applied={T|T < 7297599357563482112}]
20:07:48.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.211977 16732 catalog_manager.cc:1382] Generated new cluster ID: f2a23f42878842b8ae2d380392014a83
20:07:48.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.212244 16732 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
20:07:48.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.211771 16702 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "23fb1b979706471688cc541e571209ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:48.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.211678 16717 sys_catalog.cc:455] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23fb1b979706471688cc541e571209ba. Latest consensus state: current_term: 1 leader_uuid: "23fb1b979706471688cc541e571209ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:48.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.212505 16702 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: This master's current role is: LEADER
20:07:48.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.212597 16717 sys_catalog.cc:458] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [sys.catalog]: This master's current role is: FOLLOWER
20:07:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.240972 16717 sys_catalog.cc:455] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "23fb1b979706471688cc541e571209ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:48.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.241957 16717 sys_catalog.cc:458] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73 [sys.catalog]: This master's current role is: FOLLOWER
20:07:48.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.257633 16732 catalog_manager.cc:1405] Generated new certificate authority record
20:07:48.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.259868 16732 catalog_manager.cc:1539] Loading token signing keys...
20:07:48.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.300141 16732 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba: Generated new TSK 0
20:07:48.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.303930 16732 catalog_manager.cc:1549] Initializing in-progress tserver states...
20:07:48.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.312673 16538 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } attempt: 7
20:07:48.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.352658 16538 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3: Bootstrap starting.
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.355765 16458 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/wal
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/data
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/logs
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/data/info.pb
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.129:0
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.18.129
20:07:48.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.129
20:07:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:48.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:07:48.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.362051 16538 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3: Neither blocks nor log segments found. Creating new log.
20:07:48.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.365401 16538 log.cc:826] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3: Log is configured to *not* fsync() on all Append() calls
20:07:48.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.378686 16538 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3: No bootstrap required, opened a new log
20:07:48.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.416242 16538 raft_consensus.cc:359] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:48.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.417976 16538 raft_consensus.cc:385] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:07:48.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.418493 16538 raft_consensus.cc:740] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: da179f21744a413ca9ee8255d492a8d3, State: Initialized, Role: FOLLOWER
20:07:48.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.420006 16538 consensus_queue.cc:260] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } }
20:07:48.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.437086 16538 sys_catalog.cc:565] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [sys.catalog]: configured and running, proceeding with master startup.
20:07:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.437294 16513 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23fb1b979706471688cc541e571209ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da179f21744a413ca9ee8255d492a8d3" is_pre_election: true
20:07:48.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.439291 16511 raft_consensus.cc:3060] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [term 0 FOLLOWER]: Advancing to term 1
20:07:48.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.438083 16512 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "23fb1b979706471688cc541e571209ba" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "da179f21744a413ca9ee8255d492a8d3"
20:07:48.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.436743 16742 sys_catalog.cc:455] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:48.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.447105 16742 sys_catalog.cc:458] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [sys.catalog]: This master's current role is: FOLLOWER
20:07:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.447204 16511 raft_consensus.cc:1275] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [term 1 FOLLOWER]: Refusing update from remote peer 23fb1b979706471688cc541e571209ba: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
20:07:48.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.462529 16730 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [LEADER]: Connected to new peer: Peer: permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
20:07:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.549943 16742 sys_catalog.cc:455] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23fb1b979706471688cc541e571209ba. Latest consensus state: current_term: 1 leader_uuid: "23fb1b979706471688cc541e571209ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.550714 16742 sys_catalog.cc:458] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [sys.catalog]: This master's current role is: FOLLOWER
20:07:48.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.551837 16747 sys_catalog.cc:455] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "23fb1b979706471688cc541e571209ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:48.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.552752 16747 sys_catalog.cc:458] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3 [sys.catalog]: This master's current role is: FOLLOWER
20:07:48.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:48.584519 16759 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
20:07:49.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.016721 16728 catalog_manager.cc:1294] Loaded cluster ID: f2a23f42878842b8ae2d380392014a83
20:07:49.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.017421 16728 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: loading cluster ID for follower catalog manager: success
20:07:49.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.026468 16728 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: acquiring CA information for follower catalog manager: success
20:07:49.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.033476 16728 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 348c7a4769db4e028451bbe76fa0ab73: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:49.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:49.055783 16740 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:07:49.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:49.056491 16740 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:49.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:49.057314 16740 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:07:49.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.070078 16730 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "23fb1b979706471688cc541e571209ba" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "da179f21744a413ca9ee8255d492a8d3" member_type: VOTER last_known_addr { host: "127.16.18.190" port: 42753 } } peers { permanent_uuid: "348c7a4769db4e028451bbe76fa0ab73" member_type: VOTER last_known_addr { host: "127.16.18.189" port: 45015 } } peers { permanent_uuid: "23fb1b979706471688cc541e571209ba" member_type: VOTER last_known_addr { host: "127.16.18.188" port: 36443 } } }
20:07:49.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.072016 16730 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23fb1b979706471688cc541e571209ba [sys.catalog]: This master's current role is: LEADER
20:07:49.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:49.118148 16740 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:07:49.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:49.119549 16740 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.18.129
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.193482 16740 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/data
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/wal
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.129:0
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/data/info.pb
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.129
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.18.129
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/logs
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:07:49.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.195617 16740 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:49.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.198236 16740 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:49.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:49.218065 16768 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.637732 16759 catalog_manager.cc:1294] Loaded cluster ID: f2a23f42878842b8ae2d380392014a83
20:07:49.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.640476 16759 catalog_manager.cc:1587] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3: loading cluster ID for follower catalog manager: success
20:07:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.671010 16759 catalog_manager.cc:1609] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3: acquiring CA information for follower catalog manager: success
20:07:49.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:49.706020 16759 catalog_manager.cc:1637] T 00000000000000000000000000000000 P da179f21744a413ca9ee8255d492a8d3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
20:07:51.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:49.221304 16769 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:51.151479 16740 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.929s user 0.000s sys 0.002s
20:07:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:51.153080 16740 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.931s user 0.000s sys 0.002s
20:07:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:51.152117 16770 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1928 milliseconds
20:07:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:50.617627 16767 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 16740
20:07:51.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:51.156184 16772 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:51.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.156248 16740 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:51.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.160403 16740 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:51.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.163926 16740 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:51.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.165545 16740 hybrid_clock.cc:648] HybridClock initialized: now 1781640471165480 us; error 83 us; skew 500 ppm
20:07:51.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.166796 16740 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:51.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.178130 16740 webserver.cc:533] Webserver started at http://127.16.18.129:44479/ using document root <none> and password file <none>
20:07:51.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.179487 16740 fs_manager.cc:362] Metadata directory not provided
20:07:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.179833 16740 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:51.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.180446 16740 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:07:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.187692 16740 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/data/instance:
20:07:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29c0b5291c624d4f886ab99a7523e976"
20:07:51.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:51 on dist-test-slave-mfps"
20:07:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.189312 16740 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/wal/instance:
20:07:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29c0b5291c624d4f886ab99a7523e976"
20:07:51.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:51 on dist-test-slave-mfps"
20:07:51.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.200538 16740 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.010s sys 0.001s
20:07:51.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.209276 16779 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.210902 16740 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
20:07:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.211366 16740 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/wal
20:07:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "29c0b5291c624d4f886ab99a7523e976"
20:07:51.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:51 on dist-test-slave-mfps"
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.211879 16740 fs_report.cc:389] FS layout report
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/wal
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/wal
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/data/data
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:51.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.285421 16740 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:51.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.287683 16740 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:51.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.288324 16740 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:51.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.292346 16740 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.298970 16740 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.299320 16740 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:07:51.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.299688 16740 ts_tablet_manager.cc:616] Registered 0 tablets
20:07:51.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.299944 16740 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.574167 16740 rpc_server.cc:307] RPC server started. Bound to: 127.16.18.129:39231
20:07:51.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.574298 16891 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.18.129:39231 every 8 connection(s)
20:07:51.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.579272 16740 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/data/info.pb
20:07:51.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.587954 16458 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 16740
20:07:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.588495 16458 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-0/wal/instance
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.630584 16458 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/wal
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/data
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/logs
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/data/info.pb
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.130:0
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.18.130
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.130
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:07:51.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.646939 16893 heartbeater.cc:344] Connected to a master server at 127.16.18.188:36443
20:07:51.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.647909 16893 heartbeater.cc:461] Registering TS with master...
20:07:51.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.650104 16893 heartbeater.cc:507] Master 127.16.18.188:36443 requested a full tablet report, sending...
20:07:51.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.655248 16658 ts_manager.cc:194] Registered new tserver with Master: 29c0b5291c624d4f886ab99a7523e976 (127.16.18.129:39231)
20:07:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.659695 16658 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.18.129:44315
20:07:51.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.680895 16894 heartbeater.cc:344] Connected to a master server at 127.16.18.189:45015
20:07:51.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.681962 16894 heartbeater.cc:461] Registering TS with master...
20:07:51.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.682698 16894 heartbeater.cc:507] Master 127.16.18.189:45015 requested a full tablet report, sending...
20:07:51.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.686164 16578 ts_manager.cc:194] Registered new tserver with Master: 29c0b5291c624d4f886ab99a7523e976 (127.16.18.129:39231)
20:07:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.704170 16897 heartbeater.cc:344] Connected to a master server at 127.16.18.190:42753
20:07:51.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.704594 16897 heartbeater.cc:461] Registering TS with master...
20:07:51.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.705364 16897 heartbeater.cc:507] Master 127.16.18.190:42753 requested a full tablet report, sending...
20:07:51.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:51.709352 16503 ts_manager.cc:194] Registered new tserver with Master: 29c0b5291c624d4f886ab99a7523e976 (127.16.18.129:39231)
20:07:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:52.086851 16900 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:07:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:52.087612 16900 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:52.088291 16900 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:07:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:52.126701 16900 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:07:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:52.127748 16900 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.18.130
20:07:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:52.168602 16900 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/data
20:07:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/wal
20:07:52.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.130:0
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/data/info.pb
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.130
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.18.130
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/logs
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:07:52.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:52.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:52.170296 16900 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:52.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:52.172199 16900 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:52.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:52.187943 16910 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:52.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:52.667102 16893 heartbeater.cc:499] Master 127.16.18.188:36443 was elected leader, sending a full tablet report...
20:07:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:53.591692 16909 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 16900
20:07:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:53.930088 16909 kernel_stack_watchdog.cc:198] Thread 16900 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
20:07:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:07:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:07:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:07:53.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:07:53.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:52.190789 16911 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:53.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:53.932394 16900 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.742s user 0.535s sys 1.017s
20:07:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:53.932842 16900 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.743s user 0.535s sys 1.017s
20:07:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:53.934342 16913 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:53.939401 16912 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1740 milliseconds
20:07:53.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.939837 16900 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:53.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.941469 16900 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:53.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.944720 16900 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:53.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.946252 16900 hybrid_clock.cc:648] HybridClock initialized: now 1781640473946169 us; error 119 us; skew 500 ppm
20:07:53.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.947464 16900 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:53.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.958196 16900 webserver.cc:533] Webserver started at http://127.16.18.130:45099/ using document root <none> and password file <none>
20:07:53.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.959698 16900 fs_manager.cc:362] Metadata directory not provided
20:07:53.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.960013 16900 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:53.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.960633 16900 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:07:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.968679 16900 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/data/instance:
20:07:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b7fbe956bea47c5a26028d2e106f97d"
20:07:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:53 on dist-test-slave-mfps"
20:07:53.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.970443 16900 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/wal/instance:
20:07:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b7fbe956bea47c5a26028d2e106f97d"
20:07:53.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:53 on dist-test-slave-mfps"
20:07:53.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.981485 16900 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.012s sys 0.000s
20:07:53.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.990447 16921 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.992179 16900 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.002s
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.992659 16900 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/wal
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b7fbe956bea47c5a26028d2e106f97d"
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:53 on dist-test-slave-mfps"
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:53.993105 16900 fs_report.cc:389] FS layout report
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/wal
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/wal
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/data/data
20:07:53.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:53.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:54.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.066128 16900 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:54.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.067942 16900 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:54.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.068460 16900 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:54.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.072203 16900 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.077724 16900 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.077950 16900 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:07:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.078155 16900 ts_tablet_manager.cc:616] Registered 0 tablets
20:07:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.078311 16900 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.242179 16900 rpc_server.cc:307] RPC server started. Bound to: 127.16.18.130:44761
20:07:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.242342 17033 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.18.130:44761 every 8 connection(s)
20:07:54.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.245468 16900 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/data/info.pb
20:07:54.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.250224 16458 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 16900
20:07:54.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.250856 16458 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-1/wal/instance
20:07:54.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.271164 16458 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:54.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:54.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/wal
20:07:54.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/data
20:07:54.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:54.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/logs
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/data/info.pb
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:54.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.131:0
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.18.131
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.131
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:54.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
20:07:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.291954 17034 heartbeater.cc:344] Connected to a master server at 127.16.18.188:36443
20:07:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.292557 17034 heartbeater.cc:461] Registering TS with master...
20:07:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.294127 17034 heartbeater.cc:507] Master 127.16.18.188:36443 requested a full tablet report, sending...
20:07:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.296113 17035 heartbeater.cc:344] Connected to a master server at 127.16.18.189:45015
20:07:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.296499 17035 heartbeater.cc:461] Registering TS with master...
20:07:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.296965 17036 heartbeater.cc:344] Connected to a master server at 127.16.18.190:42753
20:07:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.297178 16658 ts_manager.cc:194] Registered new tserver with Master: 2b7fbe956bea47c5a26028d2e106f97d (127.16.18.130:44761)
20:07:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.297413 17036 heartbeater.cc:461] Registering TS with master...
20:07:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.297580 17035 heartbeater.cc:507] Master 127.16.18.189:45015 requested a full tablet report, sending...
20:07:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.298727 17036 heartbeater.cc:507] Master 127.16.18.190:42753 requested a full tablet report, sending...
20:07:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.299117 16658 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.18.130:50147
20:07:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.301942 16578 ts_manager.cc:194] Registered new tserver with Master: 2b7fbe956bea47c5a26028d2e106f97d (127.16.18.130:44761)
20:07:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.302604 16503 ts_manager.cc:194] Registered new tserver with Master: 2b7fbe956bea47c5a26028d2e106f97d (127.16.18.130:44761)
20:07:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:54.663606 17044 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:07:54.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:54.664343 17044 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:54.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:54.664904 17044 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:07:54.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:54.703853 17044 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:07:54.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:54.704885 17044 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.16.18.131
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.746825 17044 tablet_server_runner.cc:78] Tablet server non-default flags:
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.18.148:42935
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/data
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/wal
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.18.131:0
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/data/info.pb
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.18.131
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.16.18.190:42753,127.16.18.189:45015,127.16.18.188:36443
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:54.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.16.18.131
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/logs
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:07:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:54.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.748914 17044 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:54.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:54.750900 17044 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:54.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:54.766906 17050 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:55.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:55.305702 17034 heartbeater.cc:499] Master 127.16.18.188:36443 was elected leader, sending a full tablet report...
20:07:56.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:54.769184 17051 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:56.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:56.074529 17053 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:07:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:56.078589 17052 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1304 milliseconds
20:07:56.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.078682 17044 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:07:56.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.080165 17044 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:07:56.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.082512 17044 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:07:56.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.083932 17044 hybrid_clock.cc:648] HybridClock initialized: now 1781640476083853 us; error 107 us; skew 500 ppm
20:07:56.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.084794 17044 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:07:56.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.091522 17044 webserver.cc:533] Webserver started at http://127.16.18.131:45953/ using document root <none> and password file <none>
20:07:56.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.092520 17044 fs_manager.cc:362] Metadata directory not provided
20:07:56.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.092716 17044 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:07:56.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.093127 17044 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:07:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.097676 17044 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/data/instance:
20:07:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7673dbef154d437a94e24b4e4303de31"
20:07:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:56 on dist-test-slave-mfps"
20:07:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.098755 17044 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/wal/instance:
20:07:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7673dbef154d437a94e24b4e4303de31"
20:07:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:56 on dist-test-slave-mfps"
20:07:56.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.106577 17044 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
20:07:56.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.112993 17062 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:07:56.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.114619 17044 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
20:07:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.114992 17044 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/wal
20:07:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7673dbef154d437a94e24b4e4303de31"
20:07:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:07:56 on dist-test-slave-mfps"
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.115381 17044 fs_report.cc:389] FS layout report
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/wal
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/wal
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/data/data
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:07:56.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:07:56.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.169740 17044 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:07:56.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.171711 17044 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:07:56.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.172232 17044 kserver.cc:163] Server-wide thread pool size limit: 3276
20:07:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.176024 17044 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
20:07:56.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.181490 17044 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
20:07:56.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.181756 17044 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
20:07:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.182071 17044 ts_tablet_manager.cc:616] Registered 0 tablets
20:07:56.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.182232 17044 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
20:07:56.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.369566 17044 rpc_server.cc:307] RPC server started. Bound to: 127.16.18.131:43707
20:07:56.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.369655 17174 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.18.131:43707 every 8 connection(s)
20:07:56.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.373064 17044 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/data/info.pb
20:07:56.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.379319 16458 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 17044
20:07:56.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.379951 16458 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster2025326451320828942/ts-2/wal/instance
20:07:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.422190 17176 heartbeater.cc:344] Connected to a master server at 127.16.18.189:45015
20:07:56.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.422897 17176 heartbeater.cc:461] Registering TS with master...
20:07:56.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.424942 17176 heartbeater.cc:507] Master 127.16.18.189:45015 requested a full tablet report, sending...
20:07:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.428691 17175 heartbeater.cc:344] Connected to a master server at 127.16.18.188:36443
20:07:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.429131 17175 heartbeater.cc:461] Registering TS with master...
20:07:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.429710 16577 ts_manager.cc:194] Registered new tserver with Master: 7673dbef154d437a94e24b4e4303de31 (127.16.18.131:43707)
20:07:56.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.430166 17175 heartbeater.cc:507] Master 127.16.18.188:36443 requested a full tablet report, sending...
20:07:56.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.433245 16658 ts_manager.cc:194] Registered new tserver with Master: 7673dbef154d437a94e24b4e4303de31 (127.16.18.131:43707)
20:07:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.435150 16658 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.16.18.131:54595
20:07:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.451731 17177 heartbeater.cc:344] Connected to a master server at 127.16.18.190:42753
20:07:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.452174 17177 heartbeater.cc:461] Registering TS with master...
20:07:56.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.452858 17177 heartbeater.cc:507] Master 127.16.18.190:42753 requested a full tablet report, sending...
20:07:56.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.455716 16503 ts_manager.cc:194] Registered new tserver with Master: 7673dbef154d437a94e24b4e4303de31 (127.16.18.131:43707)
20:07:56.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.461895 16458 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
20:07:56.475 [DEBUG - main] (MiniKuduCluster.java:195) Response:
20:07:56.475 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
20:07:56.476 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.18.190"
port: 42753
}
bound_http_address {
host: "127.16.18.190"
port: 42783
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.18.189"
port: 45015
}
bound_http_address {
host: "127.16.18.189"
port: 46849
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.18.188"
port: 36443
}
bound_http_address {
host: "127.16.18.188"
port: 40579
}
}
}
20:07:56.477 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:07:56.478 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.16.18.129"
port: 39231
}
bound_http_address {
host: "127.16.18.129"
port: 44479
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.16.18.130"
port: 44761
}
bound_http_address {
host: "127.16.18.130"
port: 45099
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.16.18.131"
port: 43707
}
bound_http_address {
host: "127.16.18.131"
port: 45953
}
}
}
20:07:56.478 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
20:07:56.479 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster1370072119012485035/krb5kdc/krb5cc
20:07:56.480 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
20:07:56.481 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task_xrIj3/test-tmp
20:07:56.481 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
20:07:56.481 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task_xrIj3/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
20:07:56.484 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
20:07:56.882 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:07:56.882 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
20:07:56.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.886737 17186 mini_oidc.cc:150] Starting JWKS server
20:07:56.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.890684 17186 webserver.cc:533] Webserver started at http://127.0.0.1:43125/ using document root <none> and password file <none>
20:07:56.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.891175 17186 mini_oidc.cc:186] Starting OIDC Discovery server
20:07:56.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:56.893630 17186 webserver.cc:533] Webserver started at http://127.0.0.1:38859/ using document root <none> and password file <none>
20:07:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
20:07:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/principal' for realm 'KRBTEST.COM',
20:07:56.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
20:07:57.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:07:57 dist-test-slave-mfps krb5kdc[17197](info): setting up network...
20:07:57.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
20:07:57.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:07:57 dist-test-slave-mfps krb5kdc[17197](info): set up 2 sockets
20:07:57.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:07:57 dist-test-slave-mfps krb5kdc[17197](info): commencing operation
20:07:57.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
20:07:57.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:57.439594 17175 heartbeater.cc:499] Master 127.16.18.188:36443 was elected leader, sending a full tablet report...
20:07:59.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.172026 17186 mini_kdc.cc:121] Time spent starting KDC: real 2.228s user 0.003s sys 0.022s
20:07:59.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
20:07:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:07:59.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
20:07:59.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
20:07:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:07:59.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
20:07:59.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
20:07:59.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:07:59.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
20:07:59.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
20:07:59.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/test-user.keytab.
20:07:59.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/test-user.keytab.
20:07:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:07:59 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640479, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:07:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:07:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T20:07:59Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
20:07:59.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T20:07:59Z Disabled control of system clock
20:07:59.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.200.190@KRBTEST.COM; defaulting to no policy
20:07:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:59.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.200.190@KRBTEST.COM" created.
20:07:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.200.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:07:59.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.200.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:07:59.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.16.200.190@KRBTEST.COM; defaulting to no policy
20:07:59.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:59.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.16.200.190@KRBTEST.COM" created.
20:07:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:07:59.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.16.200.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:07:59.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.16.200.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:07:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:59.466758 17186 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:07:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:07:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data
20:07:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:07:59.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/logs
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/info.pb
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.190:42201
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.190
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.190
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:59.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:07:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:07:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:07:59.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:07:59.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.833006 17214 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:07:59.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.833774 17214 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:07:59.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.834415 17214 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:07:59.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.875341 17214 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:07:59.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.875733 17214 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:07:59.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.876039 17214 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:07:59.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.876255 17214 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:07:59.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.876464 17214 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:07:59.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.876674 17214 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:07:59.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.876832 17214 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43125/jwks.json
20:07:59.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:59.925328 17214 master_runner.cc:387] Master server non-default flags:
20:07:59.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:07:59.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.190
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.190:42201
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/info.pb
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.190
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:07:59.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/logs
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:07:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:59.927078 17214 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:07:59.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:07:59.928658 17214 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:07:59.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.944533 17220 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:01.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.952044 17223 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:01.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:07:59.946106 17221 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:01.317687 17222 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1367 milliseconds
20:08:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.317865 17214 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:08:01.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.319401 17214 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.322118 17214 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:01.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.323534 17214 hybrid_clock.cc:648] HybridClock initialized: now 1781640481323510 us; error 61 us; skew 500 ppm
20:08:01.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.324536 17214 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:01.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:01 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640481, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:01.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.330935 17214 init.cc:377] Logged in from keytab as kudu/127.16.200.190@KRBTEST.COM (short username kudu)
20:08:01.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.339222 17214 webserver.cc:533] Webserver started at http://127.16.200.190:41103/ using document root <none> and password file <none>
20:08:01.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.341481 17214 fs_manager.cc:362] Metadata directory not provided
20:08:01.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.341849 17214 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:01.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.342583 17214 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:08:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.349133 17214 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/instance:
20:08:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8bc9db753e8f4dc5809882a204eef58e"
20:08:01.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:01 on dist-test-slave-mfps"
20:08:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.350699 17214 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal/instance:
20:08:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8bc9db753e8f4dc5809882a204eef58e"
20:08:01.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:01 on dist-test-slave-mfps"
20:08:01.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.360917 17214 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
20:08:01.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.369232 17231 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:08:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.370874 17214 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
20:08:01.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.371316 17214 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8bc9db753e8f4dc5809882a204eef58e"
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:01 on dist-test-slave-mfps"
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.371784 17214 fs_report.cc:389] FS layout report
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/data
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:01.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.427722 17214 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:08:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.436116 17214 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.436798 17214 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.531243 17214 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.190:42201
20:08:01.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.531453 17282 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.200.190:42201 every 8 connection(s)
20:08:01.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.535225 17214 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/info.pb
20:08:01.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.540936 17186 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 17214
20:08:01.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.541666 17186 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal/instance
20:08:01.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.544787 17283 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:01.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.557132 17283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } has no permanent_uuid. Determining permanent_uuid...
20:08:01.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.200.189@KRBTEST.COM; defaulting to no policy
20:08:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:08:01.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.200.189@KRBTEST.COM" created.
20:08:01.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:01 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640481, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.190@KRBTEST.COM for kudu/127.16.200.190@KRBTEST.COM
20:08:01.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:08:01.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.200.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:08:01.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.200.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:08:01.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.613678 17283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } has no permanent_uuid. Determining permanent_uuid...
20:08:01.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:01.617816 17234 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:01.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.16.200.189@KRBTEST.COM; defaulting to no policy
20:08:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:08:01.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.16.200.189@KRBTEST.COM" created.
20:08:01.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:01.623605 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.189:39725: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:08:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.16.200.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:08:01.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.16.200.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.648350 17186 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:01.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/logs
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/info.pb
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.189:39725
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.189
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.189
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:08:01.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:08:01.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.673286 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } attempt: 1
20:08:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:01.678925 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.189:39725: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:01.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.747376 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } attempt: 2
20:08:01.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:01.752432 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.189:39725: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.843873 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } attempt: 3
20:08:01.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:01.849502 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.189:39725: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:01.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:01.993047 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } attempt: 4
20:08:01.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:01.997311 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.189:39725: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:02.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.025780 17293 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:08:02.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.026492 17293 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:02.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.027118 17293 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:08:02.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.068797 17293 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:08:02.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.069093 17293 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:08:02.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.069376 17293 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:08:02.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.069705 17293 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.069934 17293 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:08:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.070151 17293 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:02.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.070295 17293 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43125/jwks.json
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:02.118526 17293 master_runner.cc:387] Master server non-default flags:
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.189
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.189:39725
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:08:02.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/info.pb
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.189
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/logs
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:08:02.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:02.120342 17293 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:02.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:02.121605 17293 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:02.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.134917 17301 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:02.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:02.296963 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } attempt: 5
20:08:02.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.306206 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.189:39725: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:02.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:02.854728 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } attempt: 6
20:08:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.869041 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.189:39725: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:03.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.136801 17302 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:03.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:02.142436 17304 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:03.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:03.547906 17303 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1409 milliseconds
20:08:03.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.548066 17293 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:08:03.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.555620 17293 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:03.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.559353 17293 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:03.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.563076 17293 hybrid_clock.cc:648] HybridClock initialized: now 1781640483562961 us; error 83 us; skew 500 ppm
20:08:03.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.564219 17293 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:03.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:03 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640483, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:03.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.573860 17293 init.cc:377] Logged in from keytab as kudu/127.16.200.189@KRBTEST.COM (short username kudu)
20:08:03.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.581317 17293 webserver.cc:533] Webserver started at http://127.16.200.189:45409/ using document root <none> and password file <none>
20:08:03.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.582854 17293 fs_manager.cc:362] Metadata directory not provided
20:08:03.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.583179 17293 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:03.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.583827 17293 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:08:03.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.590037 17293 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/instance:
20:08:03.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b22667821c3043ac8937aa2798d5f183"
20:08:03.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:03 on dist-test-slave-mfps"
20:08:03.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.591499 17293 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal/instance:
20:08:03.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b22667821c3043ac8937aa2798d5f183"
20:08:03.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:03 on dist-test-slave-mfps"
20:08:03.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.601780 17293 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.008s sys 0.000s
20:08:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.609102 17314 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:08:03.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.610531 17293 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
20:08:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.611050 17293 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b22667821c3043ac8937aa2798d5f183"
20:08:03.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:03 on dist-test-slave-mfps"
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.611513 17293 fs_report.cc:389] FS layout report
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/data
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:03.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:03.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.668862 17293 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:08:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.679972 17293 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:03.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.680469 17293 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.774334 17293 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.189:39725
20:08:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.774422 17365 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.200.189:39725 every 8 connection(s)
20:08:03.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.777988 17293 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/info.pb
20:08:03.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.783334 17186 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 17293
20:08:03.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.783880 17186 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal/instance
20:08:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.786887 17366 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:03.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.16.200.188@KRBTEST.COM; defaulting to no policy
20:08:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:08:03.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.16.200.188@KRBTEST.COM" created.
20:08:03.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.806545 17366 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } has no permanent_uuid. Determining permanent_uuid...
20:08:03.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:03 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640483, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.189@KRBTEST.COM for kudu/127.16.200.190@KRBTEST.COM
20:08:03.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:08:03.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.200.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:08:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.16.200.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:08:03.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.853549 17366 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } has no permanent_uuid. Determining permanent_uuid...
20:08:03.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.16.200.188@KRBTEST.COM; defaulting to no policy
20:08:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:08:03.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.16.200.188@KRBTEST.COM" created.
20:08:03.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:03 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640483, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.189@KRBTEST.COM for kudu/127.16.200.189@KRBTEST.COM
20:08:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
20:08:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.16.200.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:08:03.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.16.200.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab.
20:08:03.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.888345 17366 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } has no permanent_uuid. Determining permanent_uuid...
20:08:03.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.889355 17186 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:03.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:03.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:03.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/logs
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/info.pb
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.188:42583
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.188
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.188
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:08:03.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:08:03.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:08:03.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:03.891520 17316 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:03.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:03.896508 17366 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:03.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.929682 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } attempt: 7
20:08:03.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:03 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640481, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.190@KRBTEST.COM for kudu/127.16.200.189@KRBTEST.COM
20:08:03.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.946130 17366 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 1
20:08:03.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:03.952672 17366 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:03.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:03.957175 17283 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } has no permanent_uuid. Determining permanent_uuid...
20:08:03.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:03.964512 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.021200 17366 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 2
20:08:04.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.023998 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 1
20:08:04.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.027371 17366 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.029076 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.110841 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 2
20:08:04.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.117640 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.118925 17366 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 3
20:08:04.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.124506 17366 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.203181 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 3
20:08:04.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.208666 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.267966 17366 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 4
20:08:04.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.273902 17366 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.301576 17376 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:08:04.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.302266 17376 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:04.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.303049 17376 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:08:04.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.340278 17376 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:08:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.340673 17376 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:08:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.341009 17376 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:08:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.341213 17376 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.341487 17376 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:08:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.341694 17376 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:04.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.341854 17376 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43125/jwks.json
20:08:04.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.349115 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 4
20:08:04.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.354327 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.391185 17376 master_runner.cc:387] Master server non-default flags:
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.188
20:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.188:42583
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/info.pb
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.188
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/logs
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:08:04.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:08:04.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.392982 17376 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:04.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.394392 17376 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:04.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.407812 17385 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:04.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.573499 17366 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 5
20:08:04.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.584126 17366 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:04.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:04.637784 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 5
20:08:04.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.647490 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:05.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:05.131795 17366 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 6
20:08:05.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:05.167961 17366 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:05.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:05.200133 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 6
20:08:05.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:05.232607 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:05.812310 17384 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 17376
20:08:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.084703 17384 kernel_stack_watchdog.cc:198] Thread 17376 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
20:08:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:08:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:08:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:08:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:08:06.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:04.410038 17386 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.085830 17376 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.677s user 0.000s sys 0.002s
20:08:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.086901 17387 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1676 milliseconds
20:08:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.087239 17376 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.679s user 0.001s sys 0.003s
20:08:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.088884 17388 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.088891 17376 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:08:06.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.092748 17376 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:06.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.095283 17376 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:06.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.096750 17376 hybrid_clock.cc:648] HybridClock initialized: now 1781640486096716 us; error 78 us; skew 500 ppm
20:08:06.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.097839 17376 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:06.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:06 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:06.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.106395 17376 init.cc:377] Logged in from keytab as kudu/127.16.200.188@KRBTEST.COM (short username kudu)
20:08:06.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.112550 17376 webserver.cc:533] Webserver started at http://127.16.200.188:46451/ using document root <none> and password file <none>
20:08:06.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.113498 17376 fs_manager.cc:362] Metadata directory not provided
20:08:06.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.113698 17376 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:06.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.114120 17376 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
20:08:06.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.120024 17376 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/instance:
20:08:06.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "482a35cce2ad4ae0b9dd17428fede012"
20:08:06.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:06 on dist-test-slave-mfps"
20:08:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.121590 17376 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal/instance:
20:08:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "482a35cce2ad4ae0b9dd17428fede012"
20:08:06.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:06 on dist-test-slave-mfps"
20:08:06.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.131381 17376 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.000s
20:08:06.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.139988 17403 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:08:06.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.141489 17376 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
20:08:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.141904 17376 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "482a35cce2ad4ae0b9dd17428fede012"
20:08:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:06 on dist-test-slave-mfps"
20:08:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.142228 17376 fs_report.cc:389] FS layout report
20:08:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/data
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:06.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.205346 17376 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:08:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.212617 17376 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:06.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.213148 17376 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.228598 17366 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 7
20:08:06.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.238669 17366 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:06.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.266386 17283 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } attempt: 7
20:08:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.275537 17283 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.16.200.188:42583: Network error: Client connection negotiation failed: client connection to 127.16.200.188:42583: connect: Connection refused (error 111)
20:08:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.305887 17376 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.188:42583
20:08:06.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.306001 17458 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.200.188:42583 every 8 connection(s)
20:08:06.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.308924 17376 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/info.pb
20:08:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.315622 17186 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 17376
20:08:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.316303 17186 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal/instance
20:08:06.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.317792 17459 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
20:08:06.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.333240 17459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } has no permanent_uuid. Determining permanent_uuid...
20:08:06.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:06 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640479, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.200.188@KRBTEST.COM
20:08:06.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:06 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.188@KRBTEST.COM for kudu/127.16.200.190@KRBTEST.COM
20:08:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.375144 17459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } has no permanent_uuid. Determining permanent_uuid...
20:08:06.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:06 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.188@KRBTEST.COM for kudu/127.16.200.189@KRBTEST.COM
20:08:06.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:06 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640479, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.200.189@KRBTEST.COM
20:08:06.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.402961 17459 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } has no permanent_uuid. Determining permanent_uuid...
20:08:06.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:06 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.188@KRBTEST.COM for kudu/127.16.200.188@KRBTEST.COM
20:08:06.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:06 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640479, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.200.190@KRBTEST.COM
20:08:06.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.441807 17459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Bootstrap starting.
20:08:06.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.445346 17186 external_mini_cluster.cc:955] 0 TS(s) registered with all masters
20:08:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.450953 17459 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Neither blocks nor log segments found. Creating new log.
20:08:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.453598 17459 log.cc:826] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Log is configured to *not* fsync() on all Append() calls
20:08:06.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:06.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
20:08:06.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
20:08:06.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
20:08:06.457 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.16.200.190"
port: 42201
}
bound_http_address {
host: "127.16.200.190"
port: 41103
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.16.200.189"
port: 39725
}
bound_http_address {
host: "127.16.200.189"
port: 45409
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.16.200.188"
port: 42583
}
bound_http_address {
host: "127.16.200.188"
port: 46451
}
}
}
20:08:06.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
20:08:06.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:06.458 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
20:08:06.458 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
20:08:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.460175 17459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: No bootstrap required, opened a new log
20:08:06.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.490085 17459 raft_consensus.cc:359] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:06.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.491202 17459 raft_consensus.cc:385] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:06.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.491570 17459 raft_consensus.cc:740] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 482a35cce2ad4ae0b9dd17428fede012, State: Initialized, Role: FOLLOWER
20:08:06.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.492800 17459 consensus_queue.cc:260] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:06 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:06.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
20:08:06.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.496068 17470 sys_catalog.cc:455] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } } }
20:08:06.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.496836 17470 sys_catalog.cc:458] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [sys.catalog]: This master's current role is: FOLLOWER
20:08:06.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.498162 17459 sys_catalog.cc:565] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [sys.catalog]: configured and running, proceeding with master startup.
20:08:06.503 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:06.505 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc
20:08:06.507 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:08:06.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.537717 17481 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:06.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.538144 17481 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.533669 17376 master_runner.cc:187] Error getting master registration for 127.16.200.190:42201: OK, instance_id { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" instance_seqno: 1781640481504865 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
20:08:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.558446 17376 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:08:06.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:06.719763 17409 tablet.cc:2406] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:08:06.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.900243 17470 raft_consensus.cc:493] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.900794 17470 raft_consensus.cc:515] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:06.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:06.904044 17470 leader_election.cc:290] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201), b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725)
20:08:07.508 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.200.188:42583
20:08:07.508 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
20:08:07.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:07.509624 17186 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 17376
20:08:07.545 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:07.546 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.200.189:39725
20:08:07.546 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
20:08:07.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:07.546669 17186 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 17293
20:08:07.583 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:07.583 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.16.200.190:42201
20:08:07.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
20:08:07.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:07.584336 17186 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 17214
20:08:07.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:07.616 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.200.188:42583
20:08:07.616 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:07.618960 17186 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:07.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/logs
20:08:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/info.pb
20:08:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:07.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:07.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.188:42583
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.188
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46451
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.188
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:08:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:08:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.018491 17484 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:08:08.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.019250 17484 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:08.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.020233 17484 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:08:08.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.059486 17484 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:08:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.059934 17484 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:08:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.060300 17484 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:08:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.060600 17484 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:08.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.060853 17484 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:08:08.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.061070 17484 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:08.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.061229 17484 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43125/jwks.json
20:08:08.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:08.110289 17484 master_runner.cc:387] Master server non-default flags:
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.188
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.188:42583
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/info.pb
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.188
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46451
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/logs
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:08:08.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:08:08.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:08.112645 17484 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:08.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:08.114153 17484 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.130689 17490 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:09.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.136565 17493 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:09.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:08.132979 17491 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:09.592610 17492 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1454 milliseconds
20:08:09.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.592753 17484 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:08:09.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.594349 17484 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:09.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.598742 17484 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:09.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.600690 17484 hybrid_clock.cc:648] HybridClock initialized: now 1781640489600611 us; error 95 us; skew 500 ppm
20:08:09.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.602036 17484 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:09 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640489, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:09.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.610034 17484 init.cc:377] Logged in from keytab as kudu/127.16.200.188@KRBTEST.COM (short username kudu)
20:08:09.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.617547 17484 webserver.cc:533] Webserver started at http://127.16.200.188:46451/ using document root <none> and password file <none>
20:08:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.618822 17484 fs_manager.cc:362] Metadata directory not provided
20:08:09.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.619074 17484 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:09.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.630126 17484 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.008s sys 0.001s
20:08:09.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.637238 17502 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:08:09.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.639196 17484 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.005s sys 0.000s
20:08:09.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.639953 17484 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:09.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "482a35cce2ad4ae0b9dd17428fede012"
20:08:09.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:06 on dist-test-slave-mfps"
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.642798 17484 fs_report.cc:389] FS layout report
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/wal
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/data
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:09.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.701929 17484 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:08:09.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.708338 17484 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:09.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.708921 17484 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.797475 17484 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.188:42583
20:08:09.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.797550 17553 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.200.188:42583 every 8 connection(s)
20:08:09.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.800833 17484 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-2/data/info.pb
20:08:09.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.804220 17186 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 17484
20:08:09.805 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:09.805 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.200.189:39725
20:08:09.805 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.807896 17186 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:09.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/logs
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/info.pb
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.189:39725
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.189
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45409
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.189
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:08:09.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:08:09.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:08:09.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.819242 17554 sys_catalog.cc:263] Verifying existing consensus state
20:08:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.832002 17554 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Bootstrap starting.
20:08:09.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.884603 17554 log.cc:826] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Log is configured to *not* fsync() on all Append() calls
20:08:09.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.891887 17554 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
20:08:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.892899 17554 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Bootstrap complete.
20:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.930204 17554 raft_consensus.cc:359] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:09.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.934201 17554 raft_consensus.cc:385] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
20:08:09.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.934592 17554 raft_consensus.cc:740] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 482a35cce2ad4ae0b9dd17428fede012, State: Initialized, Role: FOLLOWER
20:08:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.936151 17554 consensus_queue.cc:260] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.939846 17558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } } }
20:08:09.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.940843 17558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [sys.catalog]: This master's current role is: FOLLOWER
20:08:09.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.942754 17554 sys_catalog.cc:565] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [sys.catalog]: configured and running, proceeding with master startup.
20:08:09.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:09.965534 17508 tablet.cc:2406] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: Can't schedule compaction. Clean time has not been advanced past its initial value.
20:08:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:09.977166 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:09.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:09.977600 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.976496 17484 master_runner.cc:187] Error getting master registration for 127.16.200.190:42201: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111),
20:08:09.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:09.995194 17484 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:08:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.236909 17556 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:08:10.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.237644 17556 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.238345 17556 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:08:10.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.285200 17556 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:08:10.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.285647 17556 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:08:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.286180 17556 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:08:10.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.286528 17556 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.286878 17556 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:08:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.287321 17556 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:10.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.287619 17556 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43125/jwks.json
20:08:10.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.290630 17558 raft_consensus.cc:493] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:10.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.291365 17558 raft_consensus.cc:515] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.295291 17558 leader_election.cc:290] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201), b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725)
20:08:10.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.297361 17503 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111)
20:08:10.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.309864 17503 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201): Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111)
20:08:10.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.312347 17505 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725): Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.312906 17505 leader_election.cc:304] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 482a35cce2ad4ae0b9dd17428fede012; no voters: 8bc9db753e8f4dc5809882a204eef58e, b22667821c3043ac8937aa2798d5f183
20:08:10.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.314010 17558 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.346123 17556 master_runner.cc:387] Master server non-default flags:
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:10.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.189
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.189:39725
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:10.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/info.pb
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.189
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45409
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/logs
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:08:10.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.347852 17556 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:10.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.349109 17556 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:10.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.364660 17579 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.982609 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:10.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.983024 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:11.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.368455 17580 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:11.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:10.370379 17582 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:11.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:10.372009 17556 server_base.cc:1061] running on GCE node
20:08:11.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.721796 17556 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:11.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.725543 17556 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:11.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.727070 17556 hybrid_clock.cc:648] HybridClock initialized: now 1781640491726982 us; error 102 us; skew 500 ppm
20:08:11.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.728080 17556 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:11.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:11 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640491, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:11.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.735025 17556 init.cc:377] Logged in from keytab as kudu/127.16.200.189@KRBTEST.COM (short username kudu)
20:08:11.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.741698 17556 webserver.cc:533] Webserver started at http://127.16.200.189:45409/ using document root <none> and password file <none>
20:08:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.742780 17556 fs_manager.cc:362] Metadata directory not provided
20:08:11.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.743048 17556 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.752238 17556 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
20:08:11.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.757905 17591 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:08:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.759506 17556 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
20:08:11.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.760074 17556 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:11.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b22667821c3043ac8937aa2798d5f183"
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:03 on dist-test-slave-mfps"
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.763553 17556 fs_report.cc:389] FS layout report
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/data
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:11.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:11.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.835350 17556 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:08:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.844507 17556 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:11.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.845191 17556 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:11.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.942399 17556 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.189:39725
20:08:11.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.942462 17642 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.200.189:39725 every 8 connection(s)
20:08:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.945785 17556 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/data/info.pb
20:08:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.952796 17186 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 17556
20:08:11.953 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:11.953 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.16.200.190:42201
20:08:11.953 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.955358 17186 external_mini_cluster.cc:1372] Running /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/logs
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/info.pb
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.190:42201
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.190
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41103
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.190
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:11.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
20:08:11.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
20:08:11.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.965348 17643 sys_catalog.cc:263] Verifying existing consensus state
20:08:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260616 20:08:11.966698 17643 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:08:11.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:11.967303 17556 master.cc:562] Master@127.16.200.189:39725 shutting down...
20:08:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:11.989895 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:11.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:11.990664 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:12.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.037287 17646 raft_consensus.cc:493] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:12.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.037820 17646 raft_consensus.cc:515] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:12.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.046581 17646 leader_election.cc:290] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201), b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725)
20:08:12.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.057891 17556 master.cc:584] Master@127.16.200.189:39725 shutdown complete.
20:08:12.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.063206 17556 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
20:08:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.071543 17503 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201): Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111)
20:08:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.072256 17505 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725): Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.072664 17505 leader_election.cc:304] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 482a35cce2ad4ae0b9dd17428fede012; no voters: 8bc9db753e8f4dc5809882a204eef58e, b22667821c3043ac8937aa2798d5f183
20:08:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.073781 17646 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:08:12.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:08:12.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.385959 17645 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
20:08:12.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.386735 17645 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
20:08:12.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.387732 17645 flags.cc:432] Enabled unsafe flag: --never_fsync=true
20:08:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.426214 17645 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
20:08:12.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.426512 17645 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
20:08:12.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.426940 17645 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
20:08:12.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.427271 17645 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
20:08:12.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.427569 17645 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
20:08:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.427937 17645 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
20:08:12.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.428175 17645 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:43125/jwks.json
20:08:12.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.469972 17645 master_runner.cc:387] Master server non-default flags:
20:08:12.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
20:08:12.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.16.200.148:40353
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.16.200.190:42201,127.16.200.189:39725,127.16.200.188:42583
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/kudu.keytab
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.16.200.190
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.16.200.190:42201
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43125/jwks.json
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/info.pb
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.16.200.190
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41103
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/logs
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 19:43:21 UTC on e7f111948823
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12427
20:08:12.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
20:08:12.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.471745 17645 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
20:08:12.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:12.473364 17645 file_cache.cc:492] Constructed file cache file cache with capacity 419430
20:08:12.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.486106 17657 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.997431 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:12.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.998073 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:13.888406 17656 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 17645
20:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:12.488816 17658 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:13.968313 17656 kernel_stack_watchdog.cc:198] Thread 17645 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
20:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
20:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
20:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
20:08:13.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
20:08:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:13.970069 17659 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
20:08:13.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:13.970381 17645 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.485s user 0.536s sys 0.947s
20:08:13.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:13.970803 17645 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.485s user 0.536s sys 0.947s
20:08:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:13.975579 17660 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
20:08:13.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.975718 17645 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
20:08:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.977363 17645 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
20:08:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.980540 17645 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
20:08:13.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.982069 17645 hybrid_clock.cc:648] HybridClock initialized: now 1781640493982035 us; error 54 us; skew 500 ppm
20:08:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.983008 17645 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
20:08:13.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:13 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640493, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:13.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.989902 17645 init.cc:377] Logged in from keytab as kudu/127.16.200.190@KRBTEST.COM (short username kudu)
20:08:13.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.997612 17645 webserver.cc:533] Webserver started at http://127.16.200.190:41103/ using document root <none> and password file <none>
20:08:13.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.998667 17645 fs_manager.cc:362] Metadata directory not provided
20:08:13.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:13.998988 17645 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
20:08:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:14.001657 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:14.002028 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:14.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.009392 17645 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.002s sys 0.006s
20:08:14.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.014894 17669 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
20:08:14.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.016546 17645 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
20:08:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.016979 17645 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data,/tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:08:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8bc9db753e8f4dc5809882a204eef58e"
20:08:14.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 20:08:01 on dist-test-slave-mfps"
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.019754 17645 fs_report.cc:389] FS layout report
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/data
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
20:08:14.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
20:08:14.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.090873 17645 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
20:08:14.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.102442 17645 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
20:08:14.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.103015 17645 kserver.cc:163] Server-wide thread pool size limit: 3276
20:08:14.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.137205 17686 raft_consensus.cc:493] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:14.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.137730 17686 raft_consensus.cc:515] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:14.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.140018 17686 leader_election.cc:290] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201), b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725)
20:08:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:14.148525 17503 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201): Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111)
20:08:14.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:14.149235 17505 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725): Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.149765 17505 leader_election.cc:304] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 482a35cce2ad4ae0b9dd17428fede012; no voters: 8bc9db753e8f4dc5809882a204eef58e, b22667821c3043ac8937aa2798d5f183
20:08:14.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.151122 17686 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:08:14.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.201476 17645 rpc_server.cc:307] RPC server started. Bound to: 127.16.200.190:42201
20:08:14.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.201545 17724 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.16.200.190:42201 every 8 connection(s)
20:08:14.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.204974 17645 server_base.cc:1193] Dumped server information to /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/data/info.pb
20:08:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.210748 17186 external_mini_cluster.cc:1434] Started /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu as pid 17645
20:08:14.211 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
20:08:14.212 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.200.188:42583(127.16.200.188:42583)] connecting to peer
20:08:14.214 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.216 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.217 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.217 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.218 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.219127 17725 sys_catalog.cc:263] Verifying existing consensus state
20:08:14.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260616 20:08:14.220273 17725 master.cc:417] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:08:14.221 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.16.200.188:42583(127.16.200.188:42583)] Successfully connected to peer
20:08:14.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.220875 17645 master.cc:562] Master@127.16.200.190:42201 shutting down...
20:08:14.220 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.16.200.190:42201(127.16.200.190:42201)] Successfully connected to peer
20:08:14.226 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
20:08:14.229 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc
20:08:14.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:14 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:14.234 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:08:14.234 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
20:08:14.236 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.16.200.188
20:08:14.254 [DEBUG - kudu-nio-2] (SecurityContext.java:222) Refreshing Kerberos credentials...
20:08:14.254 [DEBUG - kudu-nio-2] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/krb5kdc/krb5cc
20:08:14.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:14 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.200.188@KRBTEST.COM
20:08:14.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:14 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:14.260 [DEBUG - kudu-nio-2] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
20:08:14.260 [INFO - kudu-nio-2] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
20:08:14.261 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.16.200.190
20:08:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:14 dist-test-slave-mfps krb5kdc[17197](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781640486, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.16.200.190@KRBTEST.COM
20:08:14.283 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xbbc81f1b, L:/127.0.0.1:45882 - R:/127.16.200.188:42583] using SASL/GSSAPI
20:08:14.295 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xcc11319c, L:/127.0.0.1:44938 - R:/127.16.200.190:42201] using SASL/GSSAPI
20:08:14.304 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Service unavailable: service kudu.master.MasterService not registered on Master
20:08:14.304 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.306 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.16.200.190:42201(127.16.200.190:42201)] handling channelInactive
20:08:14.306 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state READY due to: connection closed
20:08:14.314 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.316 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.318 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.318 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.319 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.320 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.320 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.320 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.321 [WARN - kudu-nio-4] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.333 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.335 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.338 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.338 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.338 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.339 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.340 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.340 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.340 [WARN - kudu-nio-6] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.347940 17645 master.cc:584] Master@127.16.200.190:42201 shutdown complete.
20:08:14.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:14.350425 17645 builtin_ntp.cc:688] coult not shutdown socket: Network error: shutdown error: Transport endpoint is not connected (error 107)
20:08:14.353 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.355 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.356 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.356 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.357 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.358 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-task_xrIj3/test-tmp/mini-kudu-cluster10523906075253186851/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
20:08:14.358 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.359 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.359 [WARN - kudu-nio-7] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.393 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.394 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.395 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.395 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.395 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.395 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.395 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.396 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.396 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.413 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.414 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.415 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.415 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.415 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.416 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.416 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.416 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.417 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.454 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.454 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.455 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.455 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.455 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.456 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.456 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.456 [INFO - pool-44-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.457 [WARN - pool-44-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.494 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.495 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.495 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.495 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.496 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.496 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.496 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.496 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.497 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.733 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:14.735 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:14.735 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.735 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.735 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:14.737 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.737 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.737 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:14.737 [WARN - kudu-nio-2] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.999096 17484 master_runner.cc:187] Error getting master registration for 127.16.200.190:42201: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111),
20:08:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:14.999599 17484 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:08:15.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:15.005329 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:15.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:15.005620 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:15.013 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:15.014 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:15.015 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:15.015 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:15.015 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:15.016 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:15.016 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:15.016 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:15.017 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:15.733 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:15.734 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:15.734 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:15.734 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:15.735 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:15.735 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:15.735 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:15.736 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:15.737 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:16.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:16.009560 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:16.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:16.010010 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:16.573 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:16.574 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:16.575 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:16.575 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:16.575 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:16.576 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:16.576 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:16.576 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:16.577 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:16.636409 17741 raft_consensus.cc:493] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:16.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:16.636875 17741 raft_consensus.cc:515] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:16.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:16.638530 17741 leader_election.cc:290] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201), b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725)
20:08:16.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:16.641299 17503 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111) [suppressed 5 similar messages]
20:08:16.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:16.644884 17503 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201): Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111)
20:08:16.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:16.645545 17505 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725): Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:16.645879 17505 leader_election.cc:304] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 482a35cce2ad4ae0b9dd17428fede012; no voters: 8bc9db753e8f4dc5809882a204eef58e, b22667821c3043ac8937aa2798d5f183
20:08:16.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:16.646605 17741 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:08:17.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:17.013933 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:17.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:17.014284 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:17.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:17.353102 17550 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0940 after lost signal to thread 17485
20:08:17.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:17.354004 17550 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae700 after lost signal to thread 17553
20:08:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:18.018993 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:18.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:18.019501 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:19.022629 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:19.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:19.023010 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:19 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640499, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:19.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:19.614979 17498 init.cc:295] Successfully reacquired a new kerberos TGT
20:08:19.953 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:19.954 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:19.955 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:19.955 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:19.955 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:19.955 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:19.955 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:19.956 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:19.958 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:20.026113 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:20.026427 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:20.052619 17484 master_runner.cc:187] Error getting master registration for 127.16.200.190:42201: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111),
20:08:20.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:20.053288 17484 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:08:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:21.029670 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:21.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:21.029954 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:22.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:22.033391 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:22.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:22.033797 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:22.108525 17750 raft_consensus.cc:493] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:22.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:22.109005 17750 raft_consensus.cc:515] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:22.110940 17750 leader_election.cc:290] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201), b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725)
20:08:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:22.114588 17505 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:08:22.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:22.118856 17505 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725): Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:22.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:22.119760 17503 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201): Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111)
20:08:22.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:22.120224 17503 leader_election.cc:304] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 482a35cce2ad4ae0b9dd17428fede012; no voters: 8bc9db753e8f4dc5809882a204eef58e, b22667821c3043ac8937aa2798d5f183
20:08:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:22.121085 17750 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:08:22.393 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:22.394 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:22.395 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:22.395 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:22.395 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:22.395 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:22.395 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:22.395 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:22.397 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:23.036988 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:23.037526 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:24.040797 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:24.041102 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:24.173 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:24.174 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:24.174 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:24.175 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:24.175 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:24.175 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:24.175 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:24.175 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:24.178 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:25.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:25.044725 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:25.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:25.045125 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:25.057185 17484 master_runner.cc:187] Error getting master registration for 127.16.200.190:42201: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111),
20:08:25.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:25.057690 17484 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:08:25.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:25.490573 16689 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5d80 after lost signal to thread 16620
20:08:25.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:25.491166 16689 debug-util.cc:398] Leaking SignalData structure 0x7b08000b59a0 after lost signal to thread 16692
20:08:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:26.048821 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:26.049139 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:27.052388 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:27.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:27.052731 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:27.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:27.775388 17761 raft_consensus.cc:493] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:27.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:27.775985 17761 raft_consensus.cc:515] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:27.778389 17761 leader_election.cc:290] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201), b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725)
20:08:27.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:27.789009 17503 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111) [suppressed 1 similar messages]
20:08:27.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:27.804277 17503 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201): Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111)
20:08:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:27.805395 17505 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725): Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:27.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:27.806558 17505 leader_election.cc:304] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 482a35cce2ad4ae0b9dd17428fede012; no voters: 8bc9db753e8f4dc5809882a204eef58e, b22667821c3043ac8937aa2798d5f183
20:08:27.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:27.807348 17761 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:08:28.033 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:28.034 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:28.035 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:28.035 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:28.035 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:28.036 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:28.036 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:28.036 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:28.039 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:28.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:28.057384 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:28.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:28.057857 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:28.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:28.851223 17030 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad060 after lost signal to thread 16904
20:08:28.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:28.852447 17030 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5460 after lost signal to thread 17033
20:08:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:29.062820 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:29.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:29.063463 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:29 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640509, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:29.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:29.618044 17498 init.cc:295] Successfully reacquired a new kerberos TGT
20:08:29.913 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:29.915 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:29.915 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:29.915 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:29.915 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:29.916 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:29.916 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:29.917 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:29.917 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:30.062263 17484 master_runner.cc:187] Error getting master registration for 127.16.200.190:42201: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111),
20:08:30.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:30.062950 17484 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:08:30.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:30.068341 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:30.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:30.068720 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:31.072175 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:31.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:31.072510 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:31.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:31.513396 17768 raft_consensus.cc:493] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
20:08:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:31.514726 17768 raft_consensus.cc:515] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8bc9db753e8f4dc5809882a204eef58e" member_type: VOTER last_known_addr { host: "127.16.200.190" port: 42201 } } peers { permanent_uuid: "b22667821c3043ac8937aa2798d5f183" member_type: VOTER last_known_addr { host: "127.16.200.189" port: 39725 } } peers { permanent_uuid: "482a35cce2ad4ae0b9dd17428fede012" member_type: VOTER last_known_addr { host: "127.16.200.188" port: 42583 } }
20:08:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:31.517007 17768 leader_election.cc:290] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201), b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725)
20:08:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:31.524523 17503 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 8bc9db753e8f4dc5809882a204eef58e (127.16.200.190:42201): Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111)
20:08:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:31.525514 17505 leader_election.cc:336] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer b22667821c3043ac8937aa2798d5f183 (127.16.200.189:39725): Network error: Client connection negotiation failed: client connection to 127.16.200.189:39725: connect: Connection refused (error 111)
20:08:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:31.526098 17505 leader_election.cc:304] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 482a35cce2ad4ae0b9dd17428fede012; no voters: 8bc9db753e8f4dc5809882a204eef58e, b22667821c3043ac8937aa2798d5f183
20:08:31.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:31.527088 17768 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
20:08:32.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:32.075788 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:32.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:32.076045 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:33.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:33.079141 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:33.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:33.079478 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:33.913 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:33.914 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:33.915 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:33.915 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:33.915 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:33.915 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:33.916 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:33.916 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:33.916 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:34.033 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:34.034 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:34.035 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:34.035 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:34.035 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:34.035 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:34.035 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:34.036 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:34.037 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:34.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:34.083068 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:34.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:34.083397 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:34.846776 17171 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad060 after lost signal to thread 17045
20:08:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:34.848238 17171 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba300 after lost signal to thread 17174
20:08:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:35.066761 17484 master_runner.cc:187] Error getting master registration for 127.16.200.190:42201: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111),
20:08:35.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:35.067360 17484 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:08:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:35.088263 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:35.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:35.088677 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:35.333 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:35.334 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:35.335 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:35.335 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:35.335 [INFO - kudu-nio-7] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:35.336 [DEBUG - kudu-nio-7] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:35.336 [INFO - kudu-nio-7] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:35.335 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:35.337 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:35.893 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:35.894 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:35.894 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:35.895 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:35.895 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:35.896 [INFO - kudu-nio-2] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:35.896 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:35.896 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:35.898 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:36.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:36.091952 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:36.092231 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:36.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:36.472458 16689 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0ca0 after lost signal to thread 16620
20:08:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:36.473887 16689 debug-util.cc:398] Leaking SignalData structure 0x7b080007cc80 after lost signal to thread 16692
20:08:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:37.095048 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:37.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:37.095316 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:38.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:38.099900 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:38.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:38.100345 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:38.793 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:38.794 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:38.794 [INFO - kudu-nio-3] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:38.794 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:38.795 [INFO - kudu-nio-3] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:38.795 [INFO - kudu-nio-4] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:38.795 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:38.795 [INFO - kudu-nio-4] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:38.797 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:39.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:39.103823 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:39.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:39.104154 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:39.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 20:08:39 dist-test-slave-mfps krb5kdc[17197](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781640519, etypes {rep=17 tkt=17 ses=17}, kudu/127.16.200.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
20:08:39.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:39.621687 17498 init.cc:295] Successfully reacquired a new kerberos TGT
20:08:40.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:40.071810 17484 master_runner.cc:187] Error getting master registration for 127.16.200.190:42201: Network error: Client connection negotiation failed: client connection to 127.16.200.190:42201: connect: Connection refused (error 111),
20:08:40.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:40.072331 17484 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
20:08:40.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:40.107096 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:40.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:40.107475 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:41.110766 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:41.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:41.111121 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:42.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:42.114318 17569 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 482a35cce2ad4ae0b9dd17428fede012: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
20:08:42.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:42.114702 17569 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
20:08:42.133 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.189:39725(127.16.200.189:39725)] connecting to peer
20:08:42.134 [DEBUG - pool-44-thread-1] (Connection.java:748) [peer master-127.16.200.190:42201(127.16.200.190:42201)] connecting to peer
20:08:42.134 [INFO - kudu-nio-5] (Connection.java:429) Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:42.134 [DEBUG - kudu-nio-5] (Connection.java:709) [peer master-127.16.200.189:39725(127.16.200.189:39725)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:42.135 [INFO - kudu-nio-5] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.189:39725: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725
20:08:42.135 [INFO - kudu-nio-6] (Connection.java:429) Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:42.135 [DEBUG - kudu-nio-6] (Connection.java:709) [peer master-127.16.200.190:42201(127.16.200.190:42201)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:42.135 [INFO - kudu-nio-6] (ConnectToCluster.java:385) Unable to connect to master 127.16.200.190:42201: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
20:08:42.136 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201; will retry
20:08:42.138 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27925), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [6ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [78ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [92ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [93ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [107ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [107ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [109ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [125ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [128ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [128ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [144ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [147ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [184ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [185ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [205ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [245ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [245ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [281ms] refreshing cache from master, [281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [283ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [284ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [285ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [285ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [520ms] refreshing cache from master, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [525ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [800ms] refreshing cache from master, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [802ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [803ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [804ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [804ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [1520ms] refreshing cache from master, [1521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [1521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [1522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [1523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [1523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [1525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [1525ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [2360ms] refreshing cache from master, [2361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [2361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [2362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [2363ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [2364ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [2365ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [2365ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [5740ms] refreshing cache from master, [5741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [5741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [5742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [5743ms] too many traces: truncated at 100 traces, deferred=Deferred@281127786(state=PENDING, result=null, callback=(continuation of Deferred@2000114892 after retry RPC after error@1913472607) -> (continuation of Deferred@1716087738 after retry RPC after error@2068715234) -> (continuation of Deferred@915586163 after retry RPC after error@783629120) -> (continuation of Deferred@51155351 after retry RPC after error@150896146) -> (continuation of Deferred@724318810 after retry RPC after error@751070020) -> (continuation of Deferred@1269028706 after retry RPC after error@819347178) -> (continuation of Deferred@2053254208 after retry RPC after error@1970533891) -> (continuation of Deferred@540948232 after retry RPC after error@564030603) -> (continuation of Deferred@1257587924 after retry RPC after error@542924903) -> (continuation of Deferred@1904973075 after retry RPC after error@597906845) -> (continuation of Deferred@1133541081 after retry RPC after error@817556) -> (continuation of Deferred@755124107 after retry RPC after error@347398155) -> (continuation of Deferred@1414691578 after retry RPC after error@930183485) -> (continuation of Deferred@149970503 after retry RPC after error@1006060522) -> (continuation of Deferred@2083875780 after retry RPC after error@1076973038) -> (continuation of Deferred@1245262281 after retry RPC after error@1263588215) -> (continuation of Deferred@485292690 after retry RPC after error@1459129052) -> (continuation of Deferred@1431068339 after retry RPC after error@407561222) -> (continuation of Deferred@1669969399 after retry RPC after error@1899326768) -> (continuation of Deferred@1178274915 after retry RPC after error@153008758) -> (continuation of Deferred@664178863 after retry RPC after error@2106602880) -> (continuation of Deferred@1666606661 after retry RPC after error@526888275), errback=(continuation of Deferred@2000114892 after retry RPC after error@1913472607) -> (continuation of Deferred@1716087738 after retry RPC after error@2068715234) -> (continuation of Deferred@915586163 after retry RPC after error@783629120) -> (continuation of Deferred@51155351 after retry RPC after error@150896146) -> (continuation of Deferred@724318810 after retry RPC after error@751070020) -> (continuation of Deferred@1269028706 after retry RPC after error@819347178) -> (continuation of Deferred@2053254208 after retry RPC after error@1970533891) -> (continuation of Deferred@540948232 after retry RPC after error@564030603) -> (continuation of Deferred@1257587924 after retry RPC after error@542924903) -> (continuation of Deferred@1904973075 after retry RPC after error@597906845) -> (continuation of Deferred@1133541081 after retry RPC after error@817556) -> (continuation of Deferred@755124107 after retry RPC after error@347398155) -> (continuation of Deferred@1414691578 after retry RPC after error@930183485) -> (continuation of Deferred@149970503 after retry RPC after error@1006060522) -> (continuation of Deferred@2083875780 after retry RPC after error@1076973038) -> (continuation of Deferred@1245262281 after retry RPC after error@1263588215) -> (continuation of Deferred@485292690 after retry RPC after error@1459129052) -> (continuation of Deferred@1431068339 after retry RPC after error@407561222) -> (continuation of Deferred@1669969399 after retry RPC after error@1899326768) -> (continuation of Deferred@1178274915 after retry RPC after error@153008758) -> (continuation of Deferred@664178863 after retry RPC after error@2106602880) -> (continuation of Deferred@1666606661 after retry RPC after error@526888275)))
20:08:42.140 [DEBUG - main] (Connection.java:575) [peer master-127.16.200.188:42583(127.16.200.188:42583)] disconnecting while in state READY
20:08:42.140 [DEBUG - main] (Connection.java:575) [peer master-127.16.200.190:42201(127.16.200.190:42201)] disconnecting while in state TERMINATED
20:08:42.140 [DEBUG - main] (Connection.java:575) [peer master-127.16.200.189:39725(127.16.200.189:39725)] disconnecting while in state TERMINATED
20:08:42.141 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:08:42.142 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.16.200.188:42583(127.16.200.188:42583)] handling channelInactive
20:08:42.143 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.16.200.188:42583(127.16.200.188:42583)] cleaning up while in state READY due to: connection closed
20:08:42.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:42.144110 17186 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 17484
20:08:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T20:08:43Z chronyd exiting
20:08:44.444 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
20:08:44.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:44.449438 16458 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 16740
20:08:44.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:44.479296 16458 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 16900
20:08:44.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:44.513849 16458 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 17044
20:08:44.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:44.548314 16458 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 16470
20:08:44.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:44.584353 16458 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 16540
20:08:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 20:08:44.618124 16643 connection.cc:573] client connection to 127.16.18.189:45015 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
20:08:44.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 20:08:44.618444 16458 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task_xrIj3/build/tsan/bin/kudu with pid 16615
20:08:45.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T20:08:45Z chronyd exiting
20:08:46.877 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27925), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [6ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [78ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [92ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [93ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [107ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [107ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [109ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [125ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [128ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [128ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [144ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [147ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [184ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [185ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [205ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [245ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [245ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [281ms] refreshing cache from master, [281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [283ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [284ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [285ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [285ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [520ms] refreshing cache from master, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [525ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [800ms] refreshing cache from master, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [802ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [803ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [804ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [804ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [1520ms] refreshing cache from master, [1521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [1521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [1522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [1523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [1523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [1525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [1525ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [2360ms] refreshing cache from master, [2361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [2361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [2362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [2363ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [2364ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [2365ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [2365ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [5740ms] refreshing cache from master, [5741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [5741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [5742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [5743ms] too many traces: truncated at 100 traces, deferred=Deferred@281127786(state=PENDING, result=null, callback=(continuation of Deferred@2000114892 after retry RPC after error@1913472607) -> (continuation of Deferred@1716087738 after retry RPC after error@2068715234) -> (continuation of Deferred@915586163 after retry RPC after error@783629120) -> (continuation of Deferred@51155351 after retry RPC after error@150896146) -> (continuation of Deferred@724318810 after retry RPC after error@751070020) -> (continuation of Deferred@1269028706 after retry RPC after error@819347178) -> (continuation of Deferred@2053254208 after retry RPC after error@1970533891) -> (continuation of Deferred@540948232 after retry RPC after error@564030603) -> (continuation of Deferred@1257587924 after retry RPC after error@542924903) -> (continuation of Deferred@1904973075 after retry RPC after error@597906845) -> (continuation of Deferred@1133541081 after retry RPC after error@817556) -> (continuation of Deferred@755124107 after retry RPC after error@347398155) -> (continuation of Deferred@1414691578 after retry RPC after error@930183485) -> (continuation of Deferred@149970503 after retry RPC after error@1006060522) -> (continuation of Deferred@2083875780 after retry RPC after error@1076973038) -> (continuation of Deferred@1245262281 after retry RPC after error@1263588215) -> (continuation of Deferred@485292690 after retry RPC after error@1459129052) -> (continuation of Deferred@1431068339 after retry RPC after error@407561222) -> (continuation of Deferred@1669969399 after retry RPC after error@1899326768) -> (continuation of Deferred@1178274915 after retry RPC after error@153008758) -> (continuation of Deferred@664178863 after retry RPC after error@2106602880) -> (continuation of Deferred@1666606661 after retry RPC after error@526888275), errback=(continuation of Deferred@2000114892 after retry RPC after error@1913472607) -> (continuation of Deferred@1716087738 after retry RPC after error@2068715234) -> (continuation of Deferred@915586163 after retry RPC after error@783629120) -> (continuation of Deferred@51155351 after retry RPC after error@150896146) -> (continuation of Deferred@724318810 after retry RPC after error@751070020) -> (continuation of Deferred@1269028706 after retry RPC after error@819347178) -> (continuation of Deferred@2053254208 after retry RPC after error@1970533891) -> (continuation of Deferred@540948232 after retry RPC after error@564030603) -> (continuation of Deferred@1257587924 after retry RPC after error@542924903) -> (continuation of Deferred@1904973075 after retry RPC after error@597906845) -> (continuation of Deferred@1133541081 after retry RPC after error@817556) -> (continuation of Deferred@755124107 after retry RPC after error@347398155) -> (continuation of Deferred@1414691578 after retry RPC after error@930183485) -> (continuation of Deferred@149970503 after retry RPC after error@1006060522) -> (continuation of Deferred@2083875780 after retry RPC after error@1076973038) -> (continuation of Deferred@1245262281 after retry RPC after error@1263588215) -> (continuation of Deferred@485292690 after retry RPC after error@1459129052) -> (continuation of Deferred@1431068339 after retry RPC after error@407561222) -> (continuation of Deferred@1669969399 after retry RPC after error@1899326768) -> (continuation of Deferred@1178274915 after retry RPC after error@153008758) -> (continuation of Deferred@664178863 after retry RPC after error@2106602880) -> (continuation of Deferred@1666606661 after retry RPC after error@526888275)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.200.190:42201
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=23, TimeoutTracker(timeout=30000, elapsed=27925), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [6ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [78ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [92ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Service unavailable: Service unavailable: service kudu.master.MasterService not registered on Master, [93ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Service unavailable: service kudu.master.MasterService not registered on Master, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [107ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [107ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [108ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [109ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [123ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [125ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [126ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [128ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [128ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [143ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [144ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [146ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [147ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [147ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [182ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [183ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [184ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [185ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [204ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [205ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [243ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [244ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [245ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [245ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [281ms] refreshing cache from master, [281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [281ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [283ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [283ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [284ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [285ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [285ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [520ms] refreshing cache from master, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [525ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [800ms] refreshing cache from master, [800ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [801ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [802ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [803ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [804ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [804ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [805ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [1520ms] refreshing cache from master, [1521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [1521ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [1522ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [1523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [1523ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [1525ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [1525ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [2360ms] refreshing cache from master, [2361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [2361ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [2362ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [2363ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.189:39725: Network error: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725, [2364ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.190:42201: Network error: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [2365ms] Sub RPC ConnectToMaster: received response from server master-127.16.200.188:42583: OK, [2365ms] delaying RPC due to: Service unavailable: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201, [5740ms] refreshing cache from master, [5741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.188:42583, [5741ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.189:39725, [5742ms] Sub RPC ConnectToMaster: sending RPC to server master-127.16.200.190:42201, [5743ms] too many traces: truncated at 100 traces, deferred=Deferred@281127786(state=PENDING, result=null, callback=(continuation of Deferred@2000114892 after retry RPC after error@1913472607) -> (continuation of Deferred@1716087738 after retry RPC after error@2068715234) -> (continuation of Deferred@915586163 after retry RPC after error@783629120) -> (continuation of Deferred@51155351 after retry RPC after error@150896146) -> (continuation of Deferred@724318810 after retry RPC after error@751070020) -> (continuation of Deferred@1269028706 after retry RPC after error@819347178) -> (continuation of Deferred@2053254208 after retry RPC after error@1970533891) -> (continuation of Deferred@540948232 after retry RPC after error@564030603) -> (continuation of Deferred@1257587924 after retry RPC after error@542924903) -> (continuation of Deferred@1904973075 after retry RPC after error@597906845) -> (continuation of Deferred@1133541081 after retry RPC after error@817556) -> (continuation of Deferred@755124107 after retry RPC after error@347398155) -> (continuation of Deferred@1414691578 after retry RPC after error@930183485) -> (continuation of Deferred@149970503 after retry RPC after error@1006060522) -> (continuation of Deferred@2083875780 after retry RPC after error@1076973038) -> (continuation of Deferred@1245262281 after retry RPC after error@1263588215) -> (continuation of Deferred@485292690 after retry RPC after error@1459129052) -> (continuation of Deferred@1431068339 after retry RPC after error@407561222) -> (continuation of Deferred@1669969399 after retry RPC after error@1899326768) -> (continuation of Deferred@1178274915 after retry RPC after error@153008758) -> (continuation of Deferred@664178863 after retry RPC after error@2106602880) -> (continuation of Deferred@1666606661 after retry RPC after error@526888275), errback=(continuation of Deferred@2000114892 after retry RPC after error@1913472607) -> (continuation of Deferred@1716087738 after retry RPC after error@2068715234) -> (continuation of Deferred@915586163 after retry RPC after error@783629120) -> (continuation of Deferred@51155351 after retry RPC after error@150896146) -> (continuation of Deferred@724318810 after retry RPC after error@751070020) -> (continuation of Deferred@1269028706 after retry RPC after error@819347178) -> (continuation of Deferred@2053254208 after retry RPC after error@1970533891) -> (continuation of Deferred@540948232 after retry RPC after error@564030603) -> (continuation of Deferred@1257587924 after retry RPC after error@542924903) -> (continuation of Deferred@1904973075 after retry RPC after error@597906845) -> (continuation of Deferred@1133541081 after retry RPC after error@817556) -> (continuation of Deferred@755124107 after retry RPC after error@347398155) -> (continuation of Deferred@1414691578 after retry RPC after error@930183485) -> (continuation of Deferred@149970503 after retry RPC after error@1006060522) -> (continuation of Deferred@2083875780 after retry RPC after error@1076973038) -> (continuation of Deferred@1245262281 after retry RPC after error@1263588215) -> (continuation of Deferred@485292690 after retry RPC after error@1459129052) -> (continuation of Deferred@1431068339 after retry RPC after error@407561222) -> (continuation of Deferred@1669969399 after retry RPC after error@1899326768) -> (continuation of Deferred@1178274915 after retry RPC after error@153008758) -> (continuation of Deferred@664178863 after retry RPC after error@2106602880) -> (continuation of Deferred@1666606661 after retry RPC after error@526888275)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.16.200.188:42583,127.16.200.189:39725,127.16.200.190:42201) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.189:39725(127.16.200.189:39725): Connection refused: /127.16.200.189:39725,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1519)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1377)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1428)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:545)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:484)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.16.200.190:42201(127.16.200.190:42201): Connection refused: /127.16.200.190:42201
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:604)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:597)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:573)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:506)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:650)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:643)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:132)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:326)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:342)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.16.200.190:42201
Caused by: java.net.ConnectException: Connection refused
at java.base/sun.nio.ch.Net.pollConnect(Native Method)
at java.base/sun.nio.ch.Net.pollConnectNow(Net.java:684)
at java.base/sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:784)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)