05:29:13.533 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
05:29:13.546 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskVvIgJ7/test-tmp
05:29:13.549 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:29:13.553 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskVvIgJ7/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:29:13.819 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
05:29:13.974 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:29:13.975 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:29:13.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:13.978993 374 mini_oidc.cc:152] Starting JWKS server
05:29:13.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:13.983259 374 webserver.cc:460] Webserver started at http://127.0.0.1:46409/ using document root <none> and password file <none>
05:29:13.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:13.983799 374 mini_oidc.cc:189] Starting OIDC Discovery server
05:29:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:13.986685 374 webserver.cc:460] Webserver started at http://127.0.0.1:33619/ using document root <none> and password file <none>
05:29:14.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:29:14Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:29:14.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:29:14Z Disabled control of system clock
05:29:14.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:14.036418 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/wal
05:29:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/data
05:29:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:14.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/logs
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/data/info.pb
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:43269
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:14.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json with env {}
05:29:14.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.339569 386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:14.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.340325 386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:14.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.341048 386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:14.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.371397 386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:14.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.371799 386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:14.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.372138 386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:14.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.372468 386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.372816 386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.373114 386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46409/jwks.json
05:29:14.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:14.409799 386 master_runner.cc:386] Master server non-default flags:
05:29:14.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:14.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:14.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:14.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:14.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/data
05:29:14.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/wal
05:29:14.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:14.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:14.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:14.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:14.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:14.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.190:43269
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/data/info.pb
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.190
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/logs
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:14.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:14.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:14.411369 386 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:14.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:14.413545 386 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:14.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.435097 392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.436328 393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:15.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:14.436386 395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.629664 386 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:29:15.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:15.630172 394 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:29:15.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.633682 386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:15.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.636183 386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:15.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.637511 386 hybrid_clock.cc:648] HybridClock initialized: now 1782019755637486 us; error 44 us; skew 500 ppm
05:29:15.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.638381 386 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:15.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.649250 386 webserver.cc:460] Webserver started at http://127.0.93.190:38691/ using document root <none> and password file <none>
05:29:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.650153 386 fs_manager.cc:359] Metadata directory not provided
05:29:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.650336 386 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:15.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.650727 386 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:15.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.657186 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/data/instance:
05:29:15.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ed9c050eae24bb092ed3dbeb4abade0"
05:29:15.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:15 on dist-test-slave-b7wt"
05:29:15.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.658767 386 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/wal/instance:
05:29:15.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ed9c050eae24bb092ed3dbeb4abade0"
05:29:15.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:15 on dist-test-slave-b7wt"
05:29:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.667119 386 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
05:29:15.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.672870 402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.674165 386 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
05:29:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.674438 386 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/wal
05:29:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8ed9c050eae24bb092ed3dbeb4abade0"
05:29:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:15 on dist-test-slave-b7wt"
05:29:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.674774 386 fs_report.cc:383] FS layout report
05:29:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:15.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/wal
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/wal
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/data/data
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:15.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:15.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:15.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:15.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.720276 386 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.721696 386 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:15.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.722188 386 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.797977 386 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.190:43269
05:29:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.798062 453 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.190:43269 every 8 connection(s)
05:29:15.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.802167 386 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/data/info.pb
05:29:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.810442 454 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:29:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.812534 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 386
05:29:15.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.812909 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-0/wal/instance
05:29:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.816248 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:15.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/wal
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/data
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/logs
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/data/info.pb
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:15.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:15.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:39347
05:29:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
05:29:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:15.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:15.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:15.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:15.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:15.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:15.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json with env {}
05:29:15.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.825138 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } has no permanent_uuid. Determining permanent_uuid...
05:29:15.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.858386 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } has no permanent_uuid. Determining permanent_uuid...
05:29:15.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:15.860726 403 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.189:39347: connect: Connection refused (error 111)
05:29:15.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:15.865109 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:39347: Network error: Client connection negotiation failed: client connection to 127.0.93.189:39347: connect: Connection refused (error 111)
05:29:15.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.914568 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } attempt: 1
05:29:15.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:15.918609 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:39347: Network error: Client connection negotiation failed: client connection to 127.0.93.189:39347: connect: Connection refused (error 111)
05:29:15.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:15.987026 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } attempt: 2
05:29:15.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:15.991592 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:39347: Network error: Client connection negotiation failed: client connection to 127.0.93.189:39347: connect: Connection refused (error 111)
05:29:16.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:16.083039 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } attempt: 3
05:29:16.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.087050 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:39347: Network error: Client connection negotiation failed: client connection to 127.0.93.189:39347: connect: Connection refused (error 111)
05:29:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.135620 456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:16.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.136361 456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:16.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.137054 456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.167850 456 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.168241 456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:16.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.168560 456 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.168936 456 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.169298 456 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.169545 456 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46409/jwks.json
05:29:16.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:16.205056 456 master_runner.cc:386] Master server non-default flags:
05:29:16.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:16.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/data
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/wal
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:16.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.189:39347
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/data/info.pb
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.189
05:29:16.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:16.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:16.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:16.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:16.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:16.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:16.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/logs
05:29:16.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:16.206605 456 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:16.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:16.208860 456 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:16.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.229094 468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:16.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:16.230410 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } attempt: 4
05:29:16.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.234647 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:39347: Network error: Client connection negotiation failed: client connection to 127.0.93.189:39347: connect: Connection refused (error 111)
05:29:16.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:16.534103 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } attempt: 5
05:29:16.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.538055 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:39347: Network error: Client connection negotiation failed: client connection to 127.0.93.189:39347: connect: Connection refused (error 111)
05:29:17.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.085669 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } attempt: 6
05:29:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.092979 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.189:39347: Network error: Client connection negotiation failed: client connection to 127.0.93.189:39347: connect: Connection refused (error 111)
05:29:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.229287 465 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:16.231298 456 server_base.cc:1034] running on GCE node
05:29:17.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:16.229256 466 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:17.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.435962 456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:17.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.439087 456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.440464 456 hybrid_clock.cc:648] HybridClock initialized: now 1782019757440439 us; error 67 us; skew 500 ppm
05:29:17.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.441577 456 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:17.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.449600 456 webserver.cc:460] Webserver started at http://127.0.93.189:39057/ using document root <none> and password file <none>
05:29:17.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.450451 456 fs_manager.cc:359] Metadata directory not provided
05:29:17.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.450636 456 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:17.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.451009 456 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.455664 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/data/instance:
05:29:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b50b59841e1421c91038b1929e5d5b3"
05:29:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:17 on dist-test-slave-b7wt"
05:29:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.456977 456 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/wal/instance:
05:29:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b50b59841e1421c91038b1929e5d5b3"
05:29:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:17 on dist-test-slave-b7wt"
05:29:17.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.464464 456 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
05:29:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.469632 477 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:17.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.470669 456 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
05:29:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.470927 456 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/wal
05:29:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8b50b59841e1421c91038b1929e5d5b3"
05:29:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:17 on dist-test-slave-b7wt"
05:29:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.471207 456 fs_report.cc:383] FS layout report
05:29:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:17.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/wal
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/wal
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/data/data
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:17.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.508783 456 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.510301 456 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:17.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.510733 456 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:17.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.586537 456 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.189:39347
05:29:17.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.586632 528 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.189:39347 every 8 connection(s)
05:29:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.589468 456 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/data/info.pb
05:29:17.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.593490 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 456
05:29:17.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.594017 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-1/wal/instance
05:29:17.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.596871 529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:29:17.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.597430 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:17.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/wal
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/data
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/logs
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/data/info.pb
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:17.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:32769
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:17.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json with env {}
05:29:17.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.610809 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } has no permanent_uuid. Determining permanent_uuid...
05:29:17.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.637768 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } has no permanent_uuid. Determining permanent_uuid...
05:29:17.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.654371 529 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } has no permanent_uuid. Determining permanent_uuid...
05:29:17.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.657739 481 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:17.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.666260 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:17.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.715785 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 1
05:29:17.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.719929 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.788378 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 2
05:29:17.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.792748 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:17.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.884241 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 3
05:29:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.888420 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.919565 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:17.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.920212 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:17.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.920907 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.951920 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.952304 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.952591 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.952847 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.953030 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:17.953183 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:46409/jwks.json
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.987725 531 master_runner.cc:386] Master server non-default flags:
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/data
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/wal
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:17.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.188:32769
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46409/jwks.json
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/data/info.pb
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.188
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/logs
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:17.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:17.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:17.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.989140 531 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:17.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:17.991253 531 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.016480 542 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:18.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.031909 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 4
05:29:18.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.037755 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.153640 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } attempt: 7
05:29:18.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.181679 454 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } has no permanent_uuid. Determining permanent_uuid...
05:29:18.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.187340 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:18.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.245787 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 1
05:29:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.249668 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:18.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.331072 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 2
05:29:18.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.335839 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:18.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.337195 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 5
05:29:18.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.340627 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:18.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.421319 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 3
05:29:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.426560 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:18.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.567008 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 4
05:29:18.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.571796 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:18.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.855198 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 5
05:29:18.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.860018 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:18.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.888118 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 6
05:29:18.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.895768 529 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:19.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.017447 541 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:19.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:18.017624 544 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:19.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:18.019052 531 server_base.cc:1034] running on GCE node
05:29:19.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.271145 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:19.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.274255 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:19.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.275701 531 hybrid_clock.cc:648] HybridClock initialized: now 1782019759275642 us; error 78 us; skew 500 ppm
05:29:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.276571 531 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:19.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.283716 531 webserver.cc:460] Webserver started at http://127.0.93.188:39617/ using document root <none> and password file <none>
05:29:19.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.284677 531 fs_manager.cc:359] Metadata directory not provided
05:29:19.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.284866 531 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:19.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.285254 531 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.290165 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/data/instance:
05:29:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4217e682ec1a42c29d70ff8fbe35fa11"
05:29:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:19 on dist-test-slave-b7wt"
05:29:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.291393 531 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/wal/instance:
05:29:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4217e682ec1a42c29d70ff8fbe35fa11"
05:29:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:19 on dist-test-slave-b7wt"
05:29:19.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.299921 531 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
05:29:19.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.305465 556 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.306706 531 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
05:29:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.306993 531 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/wal
05:29:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4217e682ec1a42c29d70ff8fbe35fa11"
05:29:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:19 on dist-test-slave-b7wt"
05:29:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.307396 531 fs_report.cc:383] FS layout report
05:29:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/wal
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/wal
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/data/data
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:19.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.345693 531 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.347515 531 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:19.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.347909 531 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:19.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.412506 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 6
05:29:19.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:19.420471 454 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.0.93.188:32769: Network error: Client connection negotiation failed: client connection to 127.0.93.188:32769: connect: Connection refused (error 111)
05:29:19.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.428357 531 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.188:32769
05:29:19.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.428442 609 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.188:32769 every 8 connection(s)
05:29:19.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.432009 531 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/data/info.pb
05:29:19.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.437029 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 531
05:29:19.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.437502 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/master-2/wal/instance
05:29:19.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.439294 610 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:29:19.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.458276 610 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } has no permanent_uuid. Determining permanent_uuid...
05:29:19.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.486370 610 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } has no permanent_uuid. Determining permanent_uuid...
05:29:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.498404 610 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } has no permanent_uuid. Determining permanent_uuid...
05:29:19.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.518764 610 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11: Bootstrap starting.
05:29:19.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.526011 610 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11: Neither blocks nor log segments found. Creating new log.
05:29:19.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.527894 610 log.cc:824] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11: Log is configured to *not* fsync() on all Append() calls
05:29:19.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.533002 610 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11: No bootstrap required, opened a new log
05:29:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.555707 610 raft_consensus.cc:348] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.556608 610 raft_consensus.cc:374] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:29:19.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.556905 610 raft_consensus.cc:729] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4217e682ec1a42c29d70ff8fbe35fa11, State: Initialized, Role: FOLLOWER
05:29:19.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.557950 610 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:19.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.560990 619 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:19.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.561573 619 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: This master's current role is: FOLLOWER
05:29:19.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.562376 610 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: configured and running, proceeding with master startup.
05:29:19.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:19.589355 630 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:19.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:19.589705 630 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:19.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:19.603210 562 tablet.cc:2369] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:29:19.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.584136 531 master_runner.cc:186] Error getting master registration for 127.0.93.190:43269: OK, instance_id { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" instance_seqno: 1782019755776997 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:29:19.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.619046 531 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:29:19.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.956413 529 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 7
05:29:19.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.980005 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3: Bootstrap starting.
05:29:19.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.987537 529 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3: Neither blocks nor log segments found. Creating new log.
05:29:19.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.990409 529 log.cc:824] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3: Log is configured to *not* fsync() on all Append() calls
05:29:19.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.994596 619 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:29:19.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.995386 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3: No bootstrap required, opened a new log
05:29:19.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.995115 619 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:19.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:19.997706 619 leader_election.cc:288] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8ed9c050eae24bb092ed3dbeb4abade0 (127.0.93.190:43269), 8b50b59841e1421c91038b1929e5d5b3 (127.0.93.189:39347)
05:29:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.027925 529 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:20.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.028849 529 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:29:20.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.029568 529 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8b50b59841e1421c91038b1929e5d5b3, State: Initialized, Role: FOLLOWER
05:29:20.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.030821 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:20.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.033864 634 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:20.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.034977 634 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [sys.catalog]: This master's current role is: FOLLOWER
05:29:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.035691 529 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [sys.catalog]: configured and running, proceeding with master startup.
05:29:20.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.035903 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b50b59841e1421c91038b1929e5d5b3" is_pre_election: true
05:29:20.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.037369 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4217e682ec1a42c29d70ff8fbe35fa11 in term 0.
05:29:20.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.039608 557 leader_election.cc:302] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4217e682ec1a42c29d70ff8fbe35fa11, 8b50b59841e1421c91038b1929e5d5b3; no voters:
05:29:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.041191 619 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 0 FOLLOWER]: Leader pre-election won for term 1
05:29:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.041599 619 raft_consensus.cc:482] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
05:29:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.041939 619 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 0 FOLLOWER]: Advancing to term 1
05:29:20.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.053640 619 raft_consensus.cc:504] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:20.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:20.072716 645 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:20.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:20.073452 645 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.076642 619 leader_election.cc:288] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [CANDIDATE]: Term 1 election: Requested vote from peers 8ed9c050eae24bb092ed3dbeb4abade0 (127.0.93.190:43269), 8b50b59841e1421c91038b1929e5d5b3 (127.0.93.189:39347)
05:29:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.077747 504 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8b50b59841e1421c91038b1929e5d5b3"
05:29:20.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.078336 504 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [term 0 FOLLOWER]: Advancing to term 1
05:29:20.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.086619 504 raft_consensus.cc:2454] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4217e682ec1a42c29d70ff8fbe35fa11 in term 1.
05:29:20.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.088876 557 leader_election.cc:302] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4217e682ec1a42c29d70ff8fbe35fa11, 8b50b59841e1421c91038b1929e5d5b3; no voters:
05:29:20.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.089797 619 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 1 FOLLOWER]: Leader election won for term 1
05:29:20.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.108500 619 raft_consensus.cc:686] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [term 1 LEADER]: Becoming Leader. State: Replica: 4217e682ec1a42c29d70ff8fbe35fa11, State: Running, Role: LEADER
05:29:20.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.068442 456 master_runner.cc:186] Error getting master registration for 127.0.93.190:43269: OK, instance_id { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" instance_seqno: 1782019755776997 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:29:20.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.110143 619 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:20.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.109935 456 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:29:20.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.122473 647 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4217e682ec1a42c29d70ff8fbe35fa11. Latest consensus state: current_term: 1 leader_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:20.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.123646 647 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: This master's current role is: LEADER
05:29:20.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.137238 649 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
05:29:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.146009 649 catalog_manager.cc:1484] Initializing Kudu cluster ID...
05:29:20.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.170604 504 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [term 1 FOLLOWER]: Refusing update from remote peer 4217e682ec1a42c29d70ff8fbe35fa11: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:29:20.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.172583 647 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
05:29:20.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.202137 634 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4217e682ec1a42c29d70ff8fbe35fa11. Latest consensus state: current_term: 1 leader_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.206125 650 mvcc.cc:205] Tried to move back new op lower bound from 7299152937641619456 to 7299152937430999040. Current Snapshot: MvccSnapshot[applied={T|T < 7299152937641619456}]
05:29:20.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.205438 634 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [sys.catalog]: This master's current role is: FOLLOWER
05:29:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.211629 651 mvcc.cc:205] Tried to move back new op lower bound from 7299152937641619456 to 7299152937430999040. Current Snapshot: MvccSnapshot[applied={T|T < 7299152937641619456}]
05:29:20.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.219493 634 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:20.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.220521 634 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3 [sys.catalog]: This master's current role is: FOLLOWER
05:29:20.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.225867 619 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:20.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.226708 619 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: This master's current role is: LEADER
05:29:20.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.230382 649 catalog_manager.cc:1347] Generated new cluster ID: 2ccaea3cb7fb43fbaeffcd998fcbec37
05:29:20.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.230697 649 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
05:29:20.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.276674 649 catalog_manager.cc:1370] Generated new certificate authority record
05:29:20.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.279402 649 catalog_manager.cc:1504] Loading token signing keys...
05:29:20.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.302661 649 catalog_manager.cc:5868] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11: Generated new TSK 0
05:29:20.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.312960 649 catalog_manager.cc:1514] Initializing in-progress tserver states...
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.358779 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/wal
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/data
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/logs
05:29:20.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/data/info.pb
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:20.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:29:20.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.453864 454 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } attempt: 7
05:29:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.513453 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0: Bootstrap starting.
05:29:20.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.545053 454 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0: Neither blocks nor log segments found. Creating new log.
05:29:20.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.559231 454 log.cc:824] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0: Log is configured to *not* fsync() on all Append() calls
05:29:20.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.597905 454 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0: No bootstrap required, opened a new log
05:29:20.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.725150 454 raft_consensus.cc:348] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:20.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.730626 454 raft_consensus.cc:374] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:29:20.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.730968 454 raft_consensus.cc:729] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ed9c050eae24bb092ed3dbeb4abade0, State: Initialized, Role: FOLLOWER
05:29:20.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.733392 454 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } }
05:29:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.744601 658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:20.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.745448 658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [sys.catalog]: This master's current role is: FOLLOWER
05:29:20.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.751879 427 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [term 0 FOLLOWER]: Advancing to term 1
05:29:20.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.756448 429 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" is_pre_election: true
05:29:20.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.758536 428 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8ed9c050eae24bb092ed3dbeb4abade0"
05:29:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.763376 427 raft_consensus.cc:1261] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [term 1 FOLLOWER]: Refusing update from remote peer 4217e682ec1a42c29d70ff8fbe35fa11: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
05:29:20.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.764859 454 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [sys.catalog]: configured and running, proceeding with master startup.
05:29:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.765506 619 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
05:29:20.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.878604 658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4217e682ec1a42c29d70ff8fbe35fa11. Latest consensus state: current_term: 1 leader_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.883074 658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [sys.catalog]: This master's current role is: FOLLOWER
05:29:20.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.882143 666 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:20.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.884326 666 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0 [sys.catalog]: This master's current role is: FOLLOWER
05:29:20.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:20.898401 674 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
05:29:21.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:21.053908 655 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:21.054855 655 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:21.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:21.055480 655 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:21.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.080132 645 catalog_manager.cc:1259] Loaded cluster ID: 2ccaea3cb7fb43fbaeffcd998fcbec37
05:29:21.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.080858 645 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3: loading cluster ID for follower catalog manager: success
05:29:21.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.089759 645 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3: acquiring CA information for follower catalog manager: success
05:29:21.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.097209 645 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8b50b59841e1421c91038b1929e5d5b3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:29:21.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:21.102943 655 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:21.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:21.103931 655 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.129
05:29:21.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.140389 655 tablet_server_runner.cc:78] Tablet server non-default flags:
05:29:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/data
05:29:21.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/wal
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.129:0
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/data/info.pb
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.129
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.129
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/logs
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:21.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:21.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.141925 655 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:21.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.143989 655 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:21.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:21.170835 680 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:21.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.296293 619 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8ed9c050eae24bb092ed3dbeb4abade0" member_type: VOTER last_known_addr { host: "127.0.93.190" port: 43269 } } peers { permanent_uuid: "8b50b59841e1421c91038b1929e5d5b3" member_type: VOTER last_known_addr { host: "127.0.93.189" port: 39347 } } peers { permanent_uuid: "4217e682ec1a42c29d70ff8fbe35fa11" member_type: VOTER last_known_addr { host: "127.0.93.188" port: 32769 } } }
05:29:21.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.297302 619 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 [sys.catalog]: This master's current role is: LEADER
05:29:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.925776 674 catalog_manager.cc:1259] Loaded cluster ID: 2ccaea3cb7fb43fbaeffcd998fcbec37
05:29:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.926616 674 catalog_manager.cc:1552] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0: loading cluster ID for follower catalog manager: success
05:29:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.949317 674 catalog_manager.cc:1574] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0: acquiring CA information for follower catalog manager: success
05:29:21.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:21.959184 674 catalog_manager.cc:1602] T 00000000000000000000000000000000 P 8ed9c050eae24bb092ed3dbeb4abade0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
05:29:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:22.561580 679 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 655
05:29:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:23.571331 679 kernel_stack_watchdog.cc:198] Thread 655 stuck at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:637 for 386ms:
05:29:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
05:29:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
05:29:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
05:29:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
05:29:23.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:21.176703 681 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:23.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:23.572875 655 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 2.401s user 0.704s sys 1.121s
05:29:23.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:23.573444 655 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 2.401s user 0.705s sys 1.121s
05:29:23.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:23.574128 683 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:23.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:23.576570 682 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2402 milliseconds
05:29:23.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.576593 655 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:29:23.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.579048 655 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:23.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.604676 655 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:23.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.608431 655 hybrid_clock.cc:648] HybridClock initialized: now 1782019763608377 us; error 59 us; skew 500 ppm
05:29:23.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.609586 655 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:23.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.626252 655 webserver.cc:460] Webserver started at http://127.0.93.129:36175/ using document root <none> and password file <none>
05:29:23.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.627506 655 fs_manager.cc:359] Metadata directory not provided
05:29:23.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.627800 655 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.628363 655 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.635447 655 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/data/instance:
05:29:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "073d5d259c6349faaa06b16dc935fbc9"
05:29:23.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:23 on dist-test-slave-b7wt"
05:29:23.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.637068 655 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/wal/instance:
05:29:23.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "073d5d259c6349faaa06b16dc935fbc9"
05:29:23.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:23 on dist-test-slave-b7wt"
05:29:23.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.647938 655 fs_manager.cc:691] Time spent creating directory manager: real 0.010s user 0.009s sys 0.001s
05:29:23.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.665671 694 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:23.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.668434 655 fs_manager.cc:725] Time spent opening block manager: real 0.016s user 0.007s sys 0.006s
05:29:23.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.668843 655 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/wal
05:29:23.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "073d5d259c6349faaa06b16dc935fbc9"
05:29:23.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:23 on dist-test-slave-b7wt"
05:29:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.669310 655 fs_report.cc:383] FS layout report
05:29:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/wal
05:29:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/wal
05:29:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/data/data
05:29:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:23.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:23.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.729465 655 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.731572 655 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.732158 655 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:23.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.736115 655 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:29:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.742116 655 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:29:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.742398 655 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:29:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.742686 655 ts_tablet_manager.cc:610] Registered 0 tablets
05:29:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:23.742885 655 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:29:24.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.020641 655 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.129:39999
05:29:24.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.020758 807 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.129:39999 every 8 connection(s)
05:29:24.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.024261 655 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/data/info.pb
05:29:24.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.034889 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 655
05:29:24.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.035789 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-0/wal/instance
05:29:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.073115 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/wal
05:29:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/data
05:29:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/logs
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/data/info.pb
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:24.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:29:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.179811 809 heartbeater.cc:347] Connected to a master server at 127.0.93.189:39347
05:29:24.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.180346 809 heartbeater.cc:464] Registering TS with master...
05:29:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.181510 809 heartbeater.cc:511] Master 127.0.93.189:39347 requested a full tablet report, sending...
05:29:24.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.185171 494 ts_manager.cc:194] Registered new tserver with Master: 073d5d259c6349faaa06b16dc935fbc9 (127.0.93.129:39999)
05:29:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.189239 812 heartbeater.cc:347] Connected to a master server at 127.0.93.190:43269
05:29:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.189575 812 heartbeater.cc:464] Registering TS with master...
05:29:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.190397 812 heartbeater.cc:511] Master 127.0.93.190:43269 requested a full tablet report, sending...
05:29:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.193730 419 ts_manager.cc:194] Registered new tserver with Master: 073d5d259c6349faaa06b16dc935fbc9 (127.0.93.129:39999)
05:29:24.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.203066 808 heartbeater.cc:347] Connected to a master server at 127.0.93.188:32769
05:29:24.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.203773 808 heartbeater.cc:464] Registering TS with master...
05:29:24.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.204627 808 heartbeater.cc:511] Master 127.0.93.188:32769 requested a full tablet report, sending...
05:29:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.207333 573 ts_manager.cc:194] Registered new tserver with Master: 073d5d259c6349faaa06b16dc935fbc9 (127.0.93.129:39999)
05:29:24.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.210152 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.129:51821
05:29:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:24.514494 815 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:24.514986 815 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:24.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:24.515406 815 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:24.548702 815 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:24.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:24.549537 815 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.130
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.585974 815 tablet_server_runner.cc:78] Tablet server non-default flags:
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/data
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/wal
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:24.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.130:0
05:29:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/data/info.pb
05:29:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.130
05:29:24.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.130
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/logs
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:29:24.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:24.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:24.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.587543 815 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:24.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:24.589695 815 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:24.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:24.619597 825 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:25.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:25.214839 808 heartbeater.cc:503] Master 127.0.93.188:32769 was elected leader, sending a full tablet report...
05:29:26.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:24.620265 826 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:26.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:26.419911 815 thread.cc:636] OpenStack (cloud detector) Time spent creating pthread: real 1.799s user 0.564s sys 1.062s
05:29:26.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:26.012275 824 debug-util.cc:398] Leaking SignalData structure 0x7b0800014860 after lost signal to thread 815
05:29:26.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:26.420336 815 thread.cc:602] OpenStack (cloud detector) Time spent starting thread: real 1.799s user 0.564s sys 1.062s
05:29:26.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:26.421037 827 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1798 milliseconds
05:29:26.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.422398 815 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:29:26.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:26.422438 828 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:26.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.427484 815 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:26.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.429801 815 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:26.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.431181 815 hybrid_clock.cc:648] HybridClock initialized: now 1782019766431143 us; error 51 us; skew 500 ppm
05:29:26.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.432039 815 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:26.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.444801 815 webserver.cc:460] Webserver started at http://127.0.93.130:36121/ using document root <none> and password file <none>
05:29:26.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.445959 815 fs_manager.cc:359] Metadata directory not provided
05:29:26.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.446228 815 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:26.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.446736 815 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.453603 815 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/data/instance:
05:29:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26b60149109642059fbf7723af5698a6"
05:29:26.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:26 on dist-test-slave-b7wt"
05:29:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.454833 815 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/wal/instance:
05:29:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26b60149109642059fbf7723af5698a6"
05:29:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:26 on dist-test-slave-b7wt"
05:29:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.462692 815 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.009s sys 0.001s
05:29:26.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.468389 836 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:26.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.469614 815 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
05:29:26.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.469977 815 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/wal
05:29:26.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26b60149109642059fbf7723af5698a6"
05:29:26.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:26 on dist-test-slave-b7wt"
05:29:26.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.470356 815 fs_report.cc:383] FS layout report
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/wal
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/wal
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/data/data
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:26.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.524319 815 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.525964 815 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.526463 815 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:26.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.529523 815 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:29:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.533658 815 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:29:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.533854 815 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:29:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.534039 815 ts_tablet_manager.cc:610] Registered 0 tablets
05:29:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.534168 815 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:29:26.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.689813 815 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.130:39509
05:29:26.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.689898 948 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.130:39509 every 8 connection(s)
05:29:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.693239 815 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/data/info.pb
05:29:26.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.697402 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 815
05:29:26.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.697908 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-1/wal/instance
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.714612 374 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/wal
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/data
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/logs
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/data/info.pb
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:26.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
05:29:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:26.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
05:29:26.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.761860 950 heartbeater.cc:347] Connected to a master server at 127.0.93.189:39347
05:29:26.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.762382 950 heartbeater.cc:464] Registering TS with master...
05:29:26.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.763444 950 heartbeater.cc:511] Master 127.0.93.189:39347 requested a full tablet report, sending...
05:29:26.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.766490 494 ts_manager.cc:194] Registered new tserver with Master: 26b60149109642059fbf7723af5698a6 (127.0.93.130:39509)
05:29:26.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.768846 949 heartbeater.cc:347] Connected to a master server at 127.0.93.188:32769
05:29:26.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.769222 949 heartbeater.cc:464] Registering TS with master...
05:29:26.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.770079 949 heartbeater.cc:511] Master 127.0.93.188:32769 requested a full tablet report, sending...
05:29:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.772321 573 ts_manager.cc:194] Registered new tserver with Master: 26b60149109642059fbf7723af5698a6 (127.0.93.130:39509)
05:29:26.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.774745 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.130:56601
05:29:26.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.780202 951 heartbeater.cc:347] Connected to a master server at 127.0.93.190:43269
05:29:26.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.780514 951 heartbeater.cc:464] Registering TS with master...
05:29:26.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.781029 951 heartbeater.cc:511] Master 127.0.93.190:43269 requested a full tablet report, sending...
05:29:26.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:26.783891 419 ts_manager.cc:194] Registered new tserver with Master: 26b60149109642059fbf7723af5698a6 (127.0.93.130:39509)
05:29:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:27.070678 956 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:27.071158 956 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:27.071673 956 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:27.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:27.103219 956 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:27.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:27.104311 956 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.0.93.131
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:27.139997 956 tablet_server_runner.cc:78] Tablet server non-default flags:
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.0.93.148:33627
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/data
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/wal
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.0.93.131:0
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:27.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/data/info.pb
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.0.93.131
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.0.93.190:43269,127.0.93.189:39347,127.0.93.188:32769
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.0.93.131
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/logs
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:27.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:27.141631 956 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:27.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:27.143354 956 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:27.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:27.166474 965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:27.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:27.778681 949 heartbeater.cc:503] Master 127.0.93.188:32769 was elected leader, sending a full tablet report...
05:29:28.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:27.166491 968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:28.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:27.168339 956 server_base.cc:1034] running on GCE node
05:29:28.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:27.166512 966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:28.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.391937 956 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:28.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.394809 956 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:28.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.396257 956 hybrid_clock.cc:648] HybridClock initialized: now 1782019768396203 us; error 72 us; skew 500 ppm
05:29:28.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.397008 956 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:28.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.403719 956 webserver.cc:460] Webserver started at http://127.0.93.131:34355/ using document root <none> and password file <none>
05:29:28.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.404629 956 fs_manager.cc:359] Metadata directory not provided
05:29:28.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.404820 956 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.405192 956 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.409955 956 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/data/instance:
05:29:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccc50ca4de504249a17f07189244c2b7"
05:29:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:28 on dist-test-slave-b7wt"
05:29:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.410972 956 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/wal/instance:
05:29:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccc50ca4de504249a17f07189244c2b7"
05:29:28.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:28 on dist-test-slave-b7wt"
05:29:28.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.418251 956 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
05:29:28.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.424142 976 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:28.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.425740 956 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:29:28.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.426160 956 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/wal
05:29:28.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ccc50ca4de504249a17f07189244c2b7"
05:29:28.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:28 on dist-test-slave-b7wt"
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.426607 956 fs_report.cc:383] FS layout report
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/wal
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/wal
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/data/data
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:28.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.490298 956 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.491972 956 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:28.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.492480 956 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:28.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.495067 956 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
05:29:28.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.499656 956 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
05:29:28.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.499881 956 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
05:29:28.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.500263 956 ts_tablet_manager.cc:610] Registered 0 tablets
05:29:28.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.500447 956 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
05:29:28.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.655903 956 rpc_server.cc:307] RPC server started. Bound to: 127.0.93.131:45527
05:29:28.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.655999 1088 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.0.93.131:45527 every 8 connection(s)
05:29:28.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.658548 956 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/data/info.pb
05:29:28.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.668524 374 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 956
05:29:28.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.669111 374 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8060717372380630009/ts-2/wal/instance
05:29:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.740943 1090 heartbeater.cc:347] Connected to a master server at 127.0.93.189:39347
05:29:28.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.741393 1090 heartbeater.cc:464] Registering TS with master...
05:29:28.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.742596 1090 heartbeater.cc:511] Master 127.0.93.189:39347 requested a full tablet report, sending...
05:29:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.743256 1089 heartbeater.cc:347] Connected to a master server at 127.0.93.188:32769
05:29:28.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.743575 1089 heartbeater.cc:464] Registering TS with master...
05:29:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.744302 1091 heartbeater.cc:347] Connected to a master server at 127.0.93.190:43269
05:29:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.744385 1089 heartbeater.cc:511] Master 127.0.93.188:32769 requested a full tablet report, sending...
05:29:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.744638 1091 heartbeater.cc:464] Registering TS with master...
05:29:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.745322 1091 heartbeater.cc:511] Master 127.0.93.190:43269 requested a full tablet report, sending...
05:29:28.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.745805 494 ts_manager.cc:194] Registered new tserver with Master: ccc50ca4de504249a17f07189244c2b7 (127.0.93.131:45527)
05:29:28.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.747136 573 ts_manager.cc:194] Registered new tserver with Master: ccc50ca4de504249a17f07189244c2b7 (127.0.93.131:45527)
05:29:28.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.747532 419 ts_manager.cc:194] Registered new tserver with Master: ccc50ca4de504249a17f07189244c2b7 (127.0.93.131:45527)
05:29:28.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.749276 573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.0.93.131:37669
05:29:28.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:28.768380 374 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
05:29:28.780 [DEBUG - main] (MiniKuduCluster.java:195) Response:
05:29:28.781 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
05:29:28.792 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.0.93.190"
port: 43269
}
bound_http_address {
host: "127.0.93.190"
port: 38691
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.0.93.189"
port: 39347
}
bound_http_address {
host: "127.0.93.189"
port: 39057
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.0.93.188"
port: 32769
}
bound_http_address {
host: "127.0.93.188"
port: 39617
}
}
}
05:29:28.798 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:29:28.802 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.0.93.129"
port: 39999
}
bound_http_address {
host: "127.0.93.129"
port: 36175
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.0.93.130"
port: 39509
}
bound_http_address {
host: "127.0.93.130"
port: 36121
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.0.93.131"
port: 45527
}
bound_http_address {
host: "127.0.93.131"
port: 34355
}
}
}
05:29:28.802 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
05:29:29.009 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
05:29:29.147 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskVvIgJ7/test-tmp
05:29:29.149 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
05:29:29.149 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskVvIgJ7/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
05:29:29.153 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
05:29:29.515 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:29.515 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
05:29:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:29.518877 1100 mini_oidc.cc:152] Starting JWKS server
05:29:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:29.522209 1100 webserver.cc:460] Webserver started at http://127.0.0.1:38527/ using document root <none> and password file <none>
05:29:29.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:29.522600 1100 mini_oidc.cc:189] Starting OIDC Discovery server
05:29:29.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:29.524528 1100 webserver.cc:460] Webserver started at http://127.0.0.1:46309/ using document root <none> and password file <none>
05:29:29.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
05:29:29.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/principal' for realm 'KRBTEST.COM',
05:29:29.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
05:29:29.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:29 dist-test-slave-b7wt krb5kdc[1112](info): setting up network...
05:29:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
05:29:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:29 dist-test-slave-b7wt krb5kdc[1112](info): set up 2 sockets
05:29:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:29 dist-test-slave-b7wt krb5kdc[1112](info): commencing operation
05:29:29.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
05:29:29.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:29.753353 1089 heartbeater.cc:503] Master 127.0.93.188:32769 was elected leader, sending a full tablet report...
05:29:30.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:30.638990 1100 mini_kdc.cc:121] Time spent starting KDC: real 1.090s user 0.001s sys 0.019s
05:29:30.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
05:29:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:29:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
05:29:30.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
05:29:30.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:29:30.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
05:29:30.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
05:29:30.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:29:30.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
05:29:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
05:29:30.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/test-user.keytab.
05:29:30.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/test-user.keytab.
05:29:30.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:30 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019770, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:30.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:29:30.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:29:30Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
05:29:30.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:29:30Z Disabled control of system clock
05:29:30.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.62@KRBTEST.COM; defaulting to no policy
05:29:30.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:29:30.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.62@KRBTEST.COM" created.
05:29:30.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:29:30.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.62.keytab.
05:29:30.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.62.keytab.
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:30.849205 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:30.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/logs
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/info.pb
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:37487
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.62.keytab
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:30.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:29:31.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.169282 1125 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:31.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.169826 1125 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:31.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.170747 1125 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.208321 1125 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:29:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.208544 1125 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:31.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.208806 1125 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.208999 1125 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.209183 1125 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.209367 1125 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.209508 1125 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38527/jwks.json
05:29:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:31.249488 1125 master_runner.cc:386] Master server non-default flags:
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.62.keytab
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:37487
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/info.pb
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/logs
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:31.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:31.250737 1125 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:31.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:31.251766 1125 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:31.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.270577 1135 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:32.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.270746 1133 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:32.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:31.273125 1125 server_base.cc:1034] running on GCE node
05:29:32.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:31.270630 1132 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:32.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.499374 1125 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:32.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.501981 1125 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:32.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.503340 1125 hybrid_clock.cc:648] HybridClock initialized: now 1782019772503311 us; error 76 us; skew 500 ppm
05:29:32.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.504164 1125 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:32.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:32 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019772, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:32.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.514607 1125 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
05:29:32.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.520143 1125 webserver.cc:460] Webserver started at http://127.1.19.62:33251/ using document root <none> and password file <none>
05:29:32.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.521174 1125 fs_manager.cc:359] Metadata directory not provided
05:29:32.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.521364 1125 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:32.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.521903 1125 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.527062 1125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/instance:
05:29:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbdfa9d4ad584ab097614982f27df125"
05:29:32.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:32 on dist-test-slave-b7wt"
05:29:32.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.528146 1125 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal/instance:
05:29:32.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbdfa9d4ad584ab097614982f27df125"
05:29:32.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:32 on dist-test-slave-b7wt"
05:29:32.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.535554 1125 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
05:29:32.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.541706 1143 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:32.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.542866 1125 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
05:29:32.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.543169 1125 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:32.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbdfa9d4ad584ab097614982f27df125"
05:29:32.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:32 on dist-test-slave-b7wt"
05:29:32.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.543467 1125 fs_report.cc:383] FS layout report
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/data
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:32.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:32.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.589176 1125 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:32.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.596601 1125 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:32.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.597151 1125 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:32.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.685261 1125 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:37487
05:29:32.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.685351 1194 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:37487 every 8 connection(s)
05:29:32.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.688547 1125 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/info.pb
05:29:32.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.693286 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 1125
05:29:32.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.693822 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal/instance
05:29:32.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.695394 1195 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:29:32.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.707515 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } has no permanent_uuid. Determining permanent_uuid...
05:29:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.61@KRBTEST.COM; defaulting to no policy
05:29:32.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:29:32.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.61@KRBTEST.COM" created.
05:29:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:32 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019772, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
05:29:32.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:29:32.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.61.keytab.
05:29:32.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.61.keytab.
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.742911 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/logs
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/info.pb
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:32.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:45551
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.61.keytab
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:32.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:29:32.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.751734 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } has no permanent_uuid. Determining permanent_uuid...
05:29:32.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:32.754129 1146 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:32.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:32.758641 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45551: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:32.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.808228 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } attempt: 1
05:29:32.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:32.812398 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45551: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:32.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.880828 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } attempt: 2
05:29:32.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:32.884828 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45551: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:32.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:32.976296 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } attempt: 3
05:29:32.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:32.980458 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45551: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.077658 1201 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.078218 1201 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:33.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.078778 1201 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.110487 1201 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:29:33.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.110718 1201 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.110966 1201 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.111166 1201 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.111351 1201 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.111531 1201 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.111667 1201 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38527/jwks.json
05:29:33.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:33.123934 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } attempt: 4
05:29:33.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.127612 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45551: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:33.148423 1201 master_runner.cc:386] Master server non-default flags:
05:29:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:33.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.61.keytab
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:45551
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/info.pb
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:33.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/logs
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:33.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:33.149621 1201 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:33.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:33.150774 1201 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.169853 1208 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:33.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:33.427016 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } attempt: 5
05:29:33.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.431006 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45551: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:33.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:33.978446 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } attempt: 6
05:29:33.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.986284 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.61:45551: Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:34.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.170975 1211 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:34.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:33.171223 1209 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:34.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:33.171801 1201 server_base.cc:1034] running on GCE node
05:29:34.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.420292 1201 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.423264 1201 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.424674 1201 hybrid_clock.cc:648] HybridClock initialized: now 1782019774424636 us; error 62 us; skew 500 ppm
05:29:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.425503 1201 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:34.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:34 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019774, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:34.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.431326 1201 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
05:29:34.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.437491 1201 webserver.cc:460] Webserver started at http://127.1.19.61:36515/ using document root <none> and password file <none>
05:29:34.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.438442 1201 fs_manager.cc:359] Metadata directory not provided
05:29:34.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.438675 1201 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.439175 1201 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.444972 1201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/instance:
05:29:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4561a5fac30f40bfa0ebfe28bbae8da8"
05:29:34.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:34 on dist-test-slave-b7wt"
05:29:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.446277 1201 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal/instance:
05:29:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4561a5fac30f40bfa0ebfe28bbae8da8"
05:29:34.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:34 on dist-test-slave-b7wt"
05:29:34.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.453749 1201 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.004s sys 0.004s
05:29:34.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.459856 1223 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:34.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.461331 1201 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:29:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.461794 1201 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4561a5fac30f40bfa0ebfe28bbae8da8"
05:29:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:34 on dist-test-slave-b7wt"
05:29:34.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.462114 1201 fs_report.cc:383] FS layout report
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/data
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:34.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.532444 1201 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.539445 1201 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:34.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.540308 1201 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:34.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.625464 1201 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:45551
05:29:34.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.625512 1275 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:45551 every 8 connection(s)
05:29:34.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.629652 1201 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/info.pb
05:29:34.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.636996 1276 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:29:34.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.640143 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 1201
05:29:34.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.640642 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal/instance
05:29:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.654985 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } has no permanent_uuid. Determining permanent_uuid...
05:29:34.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.1.19.60@KRBTEST.COM; defaulting to no policy
05:29:34.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:29:34.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.1.19.60@KRBTEST.COM" created.
05:29:34.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:34 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019774, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
05:29:34.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.763331 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } has no permanent_uuid. Determining permanent_uuid...
05:29:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
05:29:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.60.keytab.
05:29:34.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.1.19.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.60.keytab.
05:29:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:34.775334 1100 mini_kdc.cc:262] Time spent creating service keytab for kudu/127.1.19.60: real 0.132s user 0.000s sys 0.014s
05:29:34.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.777455 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/logs
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/info.pb
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:36891
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.60.keytab
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:34.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:29:34.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:34 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019774, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
05:29:34.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.797086 1276 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } has no permanent_uuid. Determining permanent_uuid...
05:29:34.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:34.799402 1225 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:34.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:34.803897 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:34.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.853906 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 1
05:29:34.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:34.857605 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:34.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:34.925972 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 2
05:29:34.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:34.930189 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.021644 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 3
05:29:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.025758 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.046670 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } attempt: 7
05:29:35.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:35 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019772, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
05:29:35.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.072587 1195 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } has no permanent_uuid. Determining permanent_uuid...
05:29:35.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.078557 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.120417 1283 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.120998 1283 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:35.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.121598 1283 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:35.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.136962 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 1
05:29:35.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.140554 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.155027 1283 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:29:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.155253 1283 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:35.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.155602 1283 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:35.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.155853 1283 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:35.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.156119 1283 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:35.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.156338 1283 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:35.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.156479 1283 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38527/jwks.json
05:29:35.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.169210 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 4
05:29:35.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.173513 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.196353 1283 master_runner.cc:386] Master server non-default flags:
05:29:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:35.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.60.keytab
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:36891
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/info.pb
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/logs
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:35.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.197567 1283 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:35.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.198509 1283 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:35.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.218266 1293 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.221935 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 2
05:29:35.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.225667 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.311106 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 3
05:29:35.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.315641 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.456034 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 4
05:29:35.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.461081 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.472913 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 5
05:29:35.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.476444 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:35.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.744654 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 5
05:29:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.749755 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:36.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.024060 1276 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 6
05:29:36.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:36.032974 1276 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:36.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.302389 1195 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } attempt: 6
05:29:36.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:36.309072 1195 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.1.19.60:36891: Network error: Client connection negotiation failed: client connection to 127.1.19.60:36891: connect: Connection refused (error 111)
05:29:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.218315 1295 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:35.221031 1283 server_base.cc:1034] running on GCE node
05:29:36.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:35.218292 1292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:36.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.523694 1283 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.526975 1283 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:36.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.528421 1283 hybrid_clock.cc:648] HybridClock initialized: now 1782019776528370 us; error 70 us; skew 500 ppm
05:29:36.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.529204 1283 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:36.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019776, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:36.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.535728 1283 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
05:29:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.541721 1283 webserver.cc:460] Webserver started at http://127.1.19.60:34785/ using document root <none> and password file <none>
05:29:36.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.542702 1283 fs_manager.cc:359] Metadata directory not provided
05:29:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.542939 1283 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:36.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.543517 1283 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
05:29:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.549018 1283 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/instance:
05:29:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "76c13a3c70364c7395e1bbc0aa5833ad"
05:29:36.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:36 on dist-test-slave-b7wt"
05:29:36.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.550295 1283 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal/instance:
05:29:36.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "76c13a3c70364c7395e1bbc0aa5833ad"
05:29:36.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:36 on dist-test-slave-b7wt"
05:29:36.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.558692 1283 fs_manager.cc:691] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
05:29:36.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.565079 1308 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:36.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.566334 1283 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
05:29:36.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.566776 1283 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:36.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "76c13a3c70364c7395e1bbc0aa5833ad"
05:29:36.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:36 on dist-test-slave-b7wt"
05:29:36.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.567149 1283 fs_report.cc:383] FS layout report
05:29:36.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/data
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:36.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:36.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.609335 1283 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.616662 1283 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:36.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.617192 1283 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.713197 1283 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:36891
05:29:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.713318 1359 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:36891 every 8 connection(s)
05:29:36.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.715924 1283 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/info.pb
05:29:36.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.717496 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 1283
05:29:36.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.718084 1100 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal/instance
05:29:36.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.723052 1360 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
05:29:36.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.740475 1360 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } has no permanent_uuid. Determining permanent_uuid...
05:29:36.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019770, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
05:29:36.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019776, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
05:29:36.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.776579 1360 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } has no permanent_uuid. Determining permanent_uuid...
05:29:36.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019776, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
05:29:36.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019770, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.61@KRBTEST.COM
05:29:36.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.809051 1360 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } has no permanent_uuid. Determining permanent_uuid...
05:29:36.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019776, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
05:29:36.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019770, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.62@KRBTEST.COM
05:29:36.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.843287 1360 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: Bootstrap starting.
05:29:36.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.850302 1360 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: Neither blocks nor log segments found. Creating new log.
05:29:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.853037 1360 log.cc:824] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: Log is configured to *not* fsync() on all Append() calls
05:29:36.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.854554 1100 external_mini_cluster.cc:930] 0 TS(s) registered with all masters
05:29:36.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.858882 1360 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: No bootstrap required, opened a new log
05:29:36.865 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:36.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
05:29:36.878 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
05:29:36.879 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
05:29:36.883 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.19.62"
port: 37487
}
bound_http_address {
host: "127.1.19.62"
port: 33251
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.19.61"
port: 45551
}
bound_http_address {
host: "127.1.19.61"
port: 36515
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.19.60"
port: 36891
}
bound_http_address {
host: "127.1.19.60"
port: 34785
}
}
}
05:29:36.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.883354 1360 raft_consensus.cc:348] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:36.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.884059 1360 raft_consensus.cc:374] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:29:36.884 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
05:29:36.885 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:36.885 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:29:36.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.884393 1360 raft_consensus.cc:729] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76c13a3c70364c7395e1bbc0aa5833ad, State: Initialized, Role: FOLLOWER
05:29:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.885542 1360 consensus_queue.cc:260] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:36.887 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:29:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.889020 1369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } } }
05:29:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.890184 1369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [sys.catalog]: This master's current role is: FOLLOWER
05:29:36.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.891052 1360 sys_catalog.cc:564] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [sys.catalog]: configured and running, proceeding with master startup.
05:29:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019776, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:36.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:29:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:36.935937 1382 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:36.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:36.936316 1382 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:36.938 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:36.940 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc
05:29:36.949 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:29:36.950 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc
05:29:36.952 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:29:36.957 [DEBUG - Time-limited test] (SecurityContext.java:161) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
05:29:36.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.921160 1283 master_runner.cc:186] Error getting master registration for 127.1.19.62:37487: OK, instance_id { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" instance_seqno: 1782019772663232 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
05:29:36.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:36.957953 1283 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:29:36.960 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
05:29:36.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
05:29:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:36 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019776, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:36.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
05:29:36.998 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:36.998 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc
05:29:36.999 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
05:29:37.000 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.60:36891
05:29:37.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
05:29:37.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:37.002492 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 1283
05:29:37.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:37.024 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.61:45551
05:29:37.024 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
05:29:37.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:37.025010 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 1201
05:29:37.046 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:37.046 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.1.19.62:37487
05:29:37.046 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
05:29:37.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:37.047150 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 1125
05:29:37.070 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:37.071 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.60:36891
05:29:37.073 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
05:29:37.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:37.075002 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:37.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/logs
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/info.pb
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:36891
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34785
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.60.keytab
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:29:37.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.381958 1387 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:37.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.382512 1387 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:37.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.383088 1387 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.414724 1387 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:29:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.414952 1387 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.415228 1387 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.415422 1387 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:37.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.415629 1387 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:37.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.415853 1387 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:37.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.416042 1387 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38527/jwks.json
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:37.454600 1387 master_runner.cc:386] Master server non-default flags:
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.60.keytab
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.60
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.60:36891
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/info.pb
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.60
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34785
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/logs
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:37.456228 1387 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:37.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:37.457423 1387 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:37.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.481545 1394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:38.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.481725 1393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:37.483400 1396 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:38.731195 1395 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:29:38.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.731266 1387 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:29:38.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.736182 1387 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:38.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.738668 1387 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.740036 1387 hybrid_clock.cc:648] HybridClock initialized: now 1782019778739965 us; error 54 us; skew 500 ppm
05:29:38.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.740869 1387 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:38 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019778, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:38.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.751101 1387 init.cc:377] Logged in from keytab as kudu/127.1.19.60@KRBTEST.COM (short username kudu)
05:29:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.756299 1387 webserver.cc:460] Webserver started at http://127.1.19.60:34785/ using document root <none> and password file <none>
05:29:38.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.757177 1387 fs_manager.cc:359] Metadata directory not provided
05:29:38.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.757372 1387 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:38.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.765409 1387 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.003s sys 0.001s
05:29:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.770741 1405 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:38.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.771911 1387 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.001s sys 0.004s
05:29:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.772291 1387 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "76c13a3c70364c7395e1bbc0aa5833ad"
05:29:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:36 on dist-test-slave-b7wt"
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.774622 1387 fs_report.cc:383] FS layout report
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/wal
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/data
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:38.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.807552 1387 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:38.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.814486 1387 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:38.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.814913 1387 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:38.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.886416 1387 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.60:36891
05:29:38.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.886531 1456 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.60:36891 every 8 connection(s)
05:29:38.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.889485 1387 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-2/data/info.pb
05:29:38.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.893882 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 1387
05:29:38.894 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:38.894 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.61:45551
05:29:38.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.896839 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/logs
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/info.pb
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:45551
05:29:38.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36515
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.61.keytab
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:38.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:29:38.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.903344 1457 sys_catalog.cc:263] Verifying existing consensus state
05:29:38.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.908859 1457 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: Bootstrap starting.
05:29:38.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.942247 1457 log.cc:824] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: Log is configured to *not* fsync() on all Append() calls
05:29:38.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.946786 1457 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
05:29:38.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.947587 1457 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: Bootstrap complete.
05:29:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.974193 1457 raft_consensus.cc:348] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:38.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.976841 1457 raft_consensus.cc:374] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
05:29:38.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.977042 1457 raft_consensus.cc:729] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76c13a3c70364c7395e1bbc0aa5833ad, State: Initialized, Role: FOLLOWER
05:29:38.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.977874 1457 consensus_queue.cc:260] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:38.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.981096 1461 sys_catalog.cc:455] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } } }
05:29:38.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.982277 1461 sys_catalog.cc:458] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [sys.catalog]: This master's current role is: FOLLOWER
05:29:38.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:38.982821 1457 sys_catalog.cc:564] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [sys.catalog]: configured and running, proceeding with master startup.
05:29:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.016785 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.017266 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:39.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.009030 1387 master_runner.cc:186] Error getting master registration for 127.1.19.62:37487: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111),
05:29:39.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.038383 1387 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:29:39.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.071412 1411 tablet.cc:2369] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: Can't schedule compaction. Clean time has not been advanced past its initial value.
05:29:39.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.247131 1459 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:39.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.247977 1459 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:39.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.248848 1459 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.282893 1459 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:29:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.283172 1459 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.283455 1459 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:39.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.283658 1459 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:39.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.283843 1459 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:39.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.284211 1459 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:39.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.284386 1459 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38527/jwks.json
05:29:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.322409 1459 master_runner.cc:386] Master server non-default flags:
05:29:39.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.61.keytab
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.61
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.61:45551
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/info.pb
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.61
05:29:39.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36515
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/logs
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:39.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.323777 1459 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:39.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.325109 1459 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:39.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.351065 1480 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:39.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.469753 1461 raft_consensus.cc:482] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:29:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.470350 1461 raft_consensus.cc:504] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:39.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.475224 1408 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111)
05:29:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.483897 1461 leader_election.cc:288] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487), 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551)
05:29:39.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.491509 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487): Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111)
05:29:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.494593 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.495604 1408 leader_election.cc:302] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 76c13a3c70364c7395e1bbc0aa5833ad; no voters: 4561a5fac30f40bfa0ebfe28bbae8da8, bbdfa9d4ad584ab097614982f27df125
05:29:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:39.496853 1461 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:29:40.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:40.023833 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:40.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:40.024924 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:40.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.352813 1483 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:39.351313 1481 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:40.650527 1482 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
05:29:40.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.650597 1459 server_base.cc:1029] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
05:29:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.655638 1459 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.659068 1459 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.660519 1459 hybrid_clock.cc:648] HybridClock initialized: now 1782019780660424 us; error 83 us; skew 500 ppm
05:29:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.661569 1459 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:40.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:40 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019780, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.668684 1459 init.cc:377] Logged in from keytab as kudu/127.1.19.61@KRBTEST.COM (short username kudu)
05:29:40.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.674361 1459 webserver.cc:460] Webserver started at http://127.1.19.61:36515/ using document root <none> and password file <none>
05:29:40.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.675506 1459 fs_manager.cc:359] Metadata directory not provided
05:29:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.675840 1459 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:40.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.686080 1459 fs_manager.cc:709] Time spent opening directory manager: real 0.006s user 0.005s sys 0.003s
05:29:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.691632 1494 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:40.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.693068 1459 fs_manager.cc:725] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
05:29:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.693699 1459 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4561a5fac30f40bfa0ebfe28bbae8da8"
05:29:40.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:34 on dist-test-slave-b7wt"
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.697320 1459 fs_report.cc:383] FS layout report
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/data
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:40.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:40.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:40.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:40.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:40.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:40.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.763389 1459 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.769882 1459 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.770294 1459 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:40.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.928687 1459 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.61:45551
05:29:40.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.928778 1548 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.61:45551 every 8 connection(s)
05:29:40.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.932482 1459 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/data/info.pb
05:29:40.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.934863 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 1459
05:29:40.936 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:40.937 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.1.19.62:37487
05:29:40.937 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
05:29:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.940299 1100 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu
05:29:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data
05:29:40.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/logs
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/info.pb
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:37487
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33251
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.62.keytab
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:40.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
05:29:40.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.951076 1549 sys_catalog.cc:263] Verifying existing consensus state
05:29:40.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 05:29:40.952138 1549 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:29:40.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:40.952641 1459 master.cc:537] Master@127.1.19.61:45551 shutting down...
05:29:41.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.029987 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:41.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.030341 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:41.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.057041 1459 master.cc:552] Master@127.1.19.61:45551 shutdown complete.
05:29:41.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-1/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:29:41.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.313174 1551 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
05:29:41.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.313861 1551 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
05:29:41.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.314574 1551 flags.cc:425] Enabled unsafe flag: --never_fsync=true
05:29:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.346674 1551 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
05:29:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.346997 1551 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
05:29:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.347431 1551 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
05:29:41.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.347779 1551 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
05:29:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.348132 1551 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
05:29:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.348481 1551 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
05:29:41.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.348714 1551 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38527/jwks.json
05:29:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.348771 1557 raft_consensus.cc:482] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:29:41.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.349108 1557 raft_consensus.cc:504] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:41.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.350672 1557 leader_election.cc:288] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487), 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551)
05:29:41.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.360877 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487): Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111)
05:29:41.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.361506 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.361852 1408 leader_election.cc:302] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 76c13a3c70364c7395e1bbc0aa5833ad; no voters: 4561a5fac30f40bfa0ebfe28bbae8da8, bbdfa9d4ad584ab097614982f27df125
05:29:41.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.362583 1557 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.392053 1551 master_runner.cc:386] Master server non-default flags:
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.19.20:36023
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.19.62:37487,127.1.19.61:45551,127.1.19.60:36891
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/krb5kdc/kudu_127.1.19.62.keytab
05:29:41.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.1.19.62
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.19.62:37487
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38527/jwks.json
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
05:29:41.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/info.pb
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.19.62
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33251
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/logs
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 21 Jun 2026 04:19:55 UTC on ccycloud.quasar-ixeqgh.root.comops.site
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4457863
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.393327 1551 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
05:29:41.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.394590 1551 file_cache.cc:493] Constructed file cache file cache with capacity 419430
05:29:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.411586 1562 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:42.036650 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:42.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:42.037268 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:42.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:42.231443 525 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7ee0 after lost signal to thread 460
05:29:42.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:42.232581 525 debug-util.cc:398] Leaking SignalData structure 0x7b080007d460 after lost signal to thread 528
05:29:42.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.412943 1565 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:42.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:41.413484 1551 server_base.cc:1034] running on GCE node
05:29:42.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:41.411639 1563 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
05:29:42.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.680488 1551 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
05:29:42.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.683663 1551 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
05:29:42.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.685340 1551 hybrid_clock.cc:648] HybridClock initialized: now 1782019782685249 us; error 110 us; skew 500 ppm
05:29:42.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.686229 1551 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
05:29:42.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:42 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019782, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:42.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.693456 1551 init.cc:377] Logged in from keytab as kudu/127.1.19.62@KRBTEST.COM (short username kudu)
05:29:42.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.703768 1551 webserver.cc:460] Webserver started at http://127.1.19.62:33251/ using document root <none> and password file <none>
05:29:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.704988 1551 fs_manager.cc:359] Metadata directory not provided
05:29:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.705268 1551 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
05:29:42.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.714752 1551 fs_manager.cc:709] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
05:29:42.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.719230 1573 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
05:29:42.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.720443 1551 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
05:29:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.720737 1551 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data,/tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bbdfa9d4ad584ab097614982f27df125"
05:29:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-21 05:29:32 on dist-test-slave-b7wt"
05:29:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.722976 1551 fs_report.cc:383] FS layout report
05:29:42.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/data
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
05:29:42.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
05:29:42.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.768898 1551 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
05:29:42.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.774801 1551 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
05:29:42.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.775205 1551 kserver.cc:163] Server-wide thread pool size limit: 3276
05:29:42.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.882522 1551 rpc_server.cc:307] RPC server started. Bound to: 127.1.19.62:37487
05:29:42.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.882588 1624 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.1.19.62:37487 every 8 connection(s)
05:29:42.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.886883 1551 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/data/info.pb
05:29:42.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.897251 1100 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu as pid 1551
05:29:42.898 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
05:29:42.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.905198 1625 sys_catalog.cc:263] Verifying existing consensus state
05:29:42.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) E20260621 05:29:42.907349 1625 master.cc:392] Unable to init master catalog manager: Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:29:42.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:42.908061 1551 master.cc:537] Master@127.1.19.62:37487 shutting down...
05:29:42.968 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.60:36891(127.1.19.60:36891)] connecting to peer
05:29:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:43.001981 1551 master.cc:552] Master@127.1.19.62:37487 shutdown complete.
05:29:43.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Not found: Unable to initialize catalog manager: Failed to initialize sys tables async: Unable to load consensus metadata for tablet 00000000000000000000000000000000: Unable to load consensus metadata for tablet 00000000000000000000000000000000: /tmp/dist-test-taskVvIgJ7/test-tmp/mini-kudu-cluster8086280507856129809/master-0/wal/consensus-meta/00000000000000000000000000000000: No such file or directory (error 2)
05:29:43.040 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:43.040488 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:43.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:43.040793 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:43.044 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.092 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.092 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.1.19.60:36891(127.1.19.60:36891)] Successfully connected to peer
05:29:43.095 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.177 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.177 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.178 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.195 [WARN - kudu-nio-0] (SecurityContext.java:209) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
05:29:43.206 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.1.19.60
05:29:43.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:43 dist-test-slave-b7wt krb5kdc[1112](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019776, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.1.19.60@KRBTEST.COM
05:29:43.454 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xcd555b47, L:/127.0.0.1:54274 - R:/127.1.19.60:36891] using SASL/GSSAPI
05:29:43.475 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.486 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.491 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.493 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.493 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.496 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.497 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.498 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.499 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.499 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.523 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.524 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.526 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.527 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.531 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.531 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.531 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.532 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.542 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.543 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.545 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.545 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.546 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.583 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.584 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.587 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.587 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.587 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.602 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.603 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.605 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.605 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.606 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.683 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.684 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.686 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.686 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.687 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.693 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.693 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.694 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.694 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.703 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.704 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.707 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.707 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.710 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.710 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.711 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.711 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.711 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.823 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.823 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.828 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.834 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.834 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.835 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.835 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:43.944 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:43.945 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:43.947 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.948 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:43.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:43.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:44.041997 1387 master_runner.cc:186] Error getting master registration for 127.1.19.62:37487: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111),
05:29:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:44.042476 1387 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:29:44.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:44.045161 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:44.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:44.045544 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:44.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:44.292461 1633 raft_consensus.cc:482] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:29:44.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:44.292966 1633 raft_consensus.cc:504] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:44.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:44.295001 1633 leader_election.cc:288] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487), 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551)
05:29:44.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:44.300940 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487): Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111)
05:29:44.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:44.301414 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:44.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:44.301653 1408 leader_election.cc:302] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 76c13a3c70364c7395e1bbc0aa5833ad; no voters: 4561a5fac30f40bfa0ebfe28bbae8da8, bbdfa9d4ad584ab097614982f27df125
05:29:44.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:44.302222 1633 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:29:44.383 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:44.384 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:44.385 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:44.385 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:44.386 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:44.388 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:44.388 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:44.388 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:44.389 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:45.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:45.048595 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:45.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:45.048846 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:45.543 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:45.544 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:45.546 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:45.547 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:45.547 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:45.547 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:45.548 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:45.548 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:45.548 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:46.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:46.052052 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:46.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:46.052402 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:47.055354 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:47.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:47.055657 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:48.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:48.059195 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:48.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:48.059521 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:48.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:48.547930 1642 raft_consensus.cc:482] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:29:48.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:48.548516 1642 raft_consensus.cc:504] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:48.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:48.550668 1642 leader_election.cc:288] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487), 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551)
05:29:48.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:48.553361 1408 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111) [suppressed 5 similar messages]
05:29:48.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:48.556841 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487): Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111)
05:29:48.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:48.557493 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:48.557752 1408 leader_election.cc:302] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 76c13a3c70364c7395e1bbc0aa5833ad; no voters: 4561a5fac30f40bfa0ebfe28bbae8da8, bbdfa9d4ad584ab097614982f27df125
05:29:48.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:48.558295 1642 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:29:48.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:48 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019788, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:48.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:48.755230 1401 init.cc:295] Successfully reacquired a new kerberos TGT
05:29:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:49.044963 1387 master_runner.cc:186] Error getting master registration for 127.1.19.62:37487: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111),
05:29:49.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:49.045465 1387 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:29:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:49.062296 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:49.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:49.062636 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:49.422 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:49.423 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:49.424 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:49.424 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:49.425 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:49.426 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:49.426 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:49.426 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:49.427 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:50.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:50.065689 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:50.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:50.066005 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:50.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:50.248245 525 debug-util.cc:398] Leaking SignalData structure 0x7b080009a5e0 after lost signal to thread 460
05:29:50.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:50.248919 525 debug-util.cc:398] Leaking SignalData structure 0x7b080009a520 after lost signal to thread 528
05:29:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:51.068857 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:51.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:51.069093 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:51.561929 1453 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae160 after lost signal to thread 1388
05:29:51.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:51.562461 1453 debug-util.cc:398] Leaking SignalData structure 0x7b08000977e0 after lost signal to thread 1456
05:29:51.863 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:51.867 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:51.867 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:51.868 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:51.868 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:51.872 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:51.872 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:51.873 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:51.873 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:52.072065 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:52.072386 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:53.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:53.074994 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:53.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:53.075237 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:53.623 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:53.625 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:53.625 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:53.625 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:53.626 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:53.627 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:53.627 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:53.627 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:53.627 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:54.078056 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:54.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:54.078320 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:54.086512 1387 master_runner.cc:186] Error getting master registration for 127.1.19.62:37487: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111),
05:29:54.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:54.086890 1387 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:29:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:55.081068 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:55.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:55.081318 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:55.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:55.741375 1652 raft_consensus.cc:482] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:29:55.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:55.741779 1652 raft_consensus.cc:504] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:29:55.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:55.743412 1652 leader_election.cc:288] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487), 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551)
05:29:55.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:55.746073 1408 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:29:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:55.749752 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487): Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111)
05:29:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:55.750284 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:29:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:55.750634 1408 leader_election.cc:302] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 76c13a3c70364c7395e1bbc0aa5833ad; no voters: 4561a5fac30f40bfa0ebfe28bbae8da8, bbdfa9d4ad584ab097614982f27df125
05:29:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:55.751266 1652 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:29:56.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:56.084198 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:56.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:56.084474 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:56.283 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:56.285 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:56.285 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:56.286 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:56.285 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:56.288 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:56.288 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:56.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:56.289 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:57.087357 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:57.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:57.087658 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:57.403 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:29:57.408 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:29:57.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:57.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:57.408 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:29:57.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:57.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:57.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:29:57.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:29:58.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:58.091358 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:58.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:58.091746 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:29:58.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:29:58 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019798, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:29:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:58.757879 1401 init.cc:295] Successfully reacquired a new kerberos TGT
05:29:59.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:59.090422 1387 master_runner.cc:186] Error getting master registration for 127.1.19.62:37487: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111),
05:29:59.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:29:59.090834 1387 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:29:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:59.095119 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:29:59.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:29:59.095402 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:00.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:00.098217 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:00.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:00.098457 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:01.101662 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:01.102041 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:01.123 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:30:01.125 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:30:01.126 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:01.126 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:01.127 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:01.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:01.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:01.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:01.129 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:30:01.663 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:30:01.664 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:30:01.667 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:01.667 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:01.667 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:01.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:01.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:01.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:01.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:30:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:02.105166 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:02.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:02.105412 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:03.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:03.108376 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:03.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:03.108640 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:04.093793 1387 master_runner.cc:186] Error getting master registration for 127.1.19.62:37487: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111),
05:30:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:04.094288 1387 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:30:04.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:04.111428 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:04.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:04.111758 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:04.363 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:30:04.364 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:30:04.365 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:04.366 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:04.366 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:04.367 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:04.367 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:04.367 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:04.367 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:30:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:05.072278 1669 raft_consensus.cc:482] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
05:30:05.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:05.072755 1669 raft_consensus.cc:504] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "bbdfa9d4ad584ab097614982f27df125" member_type: VOTER last_known_addr { host: "127.1.19.62" port: 37487 } } peers { permanent_uuid: "4561a5fac30f40bfa0ebfe28bbae8da8" member_type: VOTER last_known_addr { host: "127.1.19.61" port: 45551 } } peers { permanent_uuid: "76c13a3c70364c7395e1bbc0aa5833ad" member_type: VOTER last_known_addr { host: "127.1.19.60" port: 36891 } }
05:30:05.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:05.074827 1669 leader_election.cc:288] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487), 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551)
05:30:05.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:05.077536 1408 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111) [suppressed 1 similar messages]
05:30:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:05.080698 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer bbdfa9d4ad584ab097614982f27df125 (127.1.19.62:37487): Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111)
05:30:05.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:05.081308 1408 leader_election.cc:334] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: RPC error from VoteRequest() call to peer 4561a5fac30f40bfa0ebfe28bbae8da8 (127.1.19.61:45551): Network error: Client connection negotiation failed: client connection to 127.1.19.61:45551: connect: Connection refused (error 111)
05:30:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:05.081614 1408 leader_election.cc:302] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 76c13a3c70364c7395e1bbc0aa5833ad; no voters: 4561a5fac30f40bfa0ebfe28bbae8da8, bbdfa9d4ad584ab097614982f27df125
05:30:05.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:05.082290 1669 raft_consensus.cc:2726] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
05:30:05.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:05.114672 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:05.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:05.114996 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:06.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:06.118206 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:06.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:06.118531 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:07.023 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:30:07.026 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:07.026 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:07.026 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:07.026 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:30:07.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:07.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:07.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:07.029 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:30:07.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:07.121464 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:07.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:07.121774 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:08.083 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:30:08.083 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:30:08.084 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:08.084 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:08.084 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:08.084 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:08.084 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:08.085 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:08.087 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:30:08.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:08.125247 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:08.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:08.125743 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:08.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 21 05:30:08 dist-test-slave-b7wt krb5kdc[1112](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1782019808, etypes {rep=17 tkt=17 ses=17}, kudu/127.1.19.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
05:30:08.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:08.760424 1401 init.cc:295] Successfully reacquired a new kerberos TGT
05:30:09.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:09.097893 1387 master_runner.cc:186] Error getting master registration for 127.1.19.62:37487: Network error: Client connection negotiation failed: client connection to 127.1.19.62:37487: connect: Connection refused (error 111),
05:30:09.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:09.098390 1387 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
05:30:09.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:09.128604 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:09.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:09.128851 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:09.323 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:30:09.324 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:30:09.324 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:09.325 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:09.325 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:09.325 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:09.325 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:09.325 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:09.327 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:30:09.643 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:30:09.644 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:30:09.644 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:09.644 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:09.644 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:09.644 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:09.645 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:09.645 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:09.646 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:30:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:10.131238 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:10.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:10.131481 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:11.133922 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:11.134153 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:12.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:12.136773 1472 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 76c13a3c70364c7395e1bbc0aa5833ad: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
05:30:12.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:12.136998 1472 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
05:30:12.403 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.61:45551(127.1.19.61:45551)] connecting to peer
05:30:12.404 [DEBUG - pool-6-thread-1] (Connection.java:748) [peer master-127.1.19.62:37487(127.1.19.62:37487)] connecting to peer
05:30:12.404 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:12.405 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.1.19.61:45551(127.1.19.61:45551)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:12.405 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.61:45551: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551
05:30:12.405 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:12.405 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.62:37487(127.1.19.62:37487)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:12.406 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.1.19.62:37487: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
05:30:12.407 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487; will retry
05:30:12.412 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29506), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [272ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [565ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [584ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [585ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [588ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [593ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [626ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [644ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [685ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [702ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [703ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [781ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [786ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [788ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [789ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [796ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [802ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [806ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [916ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [926ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [928ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [929ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1037ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [1039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1046ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1476ms] refreshing cache from master, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [1477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [1478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1483ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [2636ms] refreshing cache from master, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [2638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [2642ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [6516ms] refreshing cache from master, [6516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [6516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [6517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [6518ms] too many traces: truncated at 100 traces, deferred=Deferred@1419081759(state=PENDING, result=null, callback=(continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614), errback=(continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614)))
05:30:12.417 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.60:36891(127.1.19.60:36891)] disconnecting while in state READY
05:30:12.418 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.62:37487(127.1.19.62:37487)] disconnecting while in state TERMINATED
05:30:12.418 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.1.19.61:45551(127.1.19.61:45551)] disconnecting while in state TERMINATED
05:30:12.424 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:30:12.426 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:30:12.426 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.1.19.60:36891(127.1.19.60:36891)] handling channelInactive
05:30:12.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.1.19.60:36891(127.1.19.60:36891)] cleaning up while in state READY due to: connection closed
05:30:12.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:12.430778 1100 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 1387
05:30:13.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:30:13Z chronyd exiting
05:30:14.665 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
05:30:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:14.667328 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 655
05:30:14.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:14.697831 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 815
05:30:14.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:14.727488 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 956
05:30:14.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:14.756840 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 386
05:30:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:14.788851 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 456
05:30:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:14.805034 558 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.0.93.190:43269: connect: Connection refused (error 111)
05:30:14.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:14.808233 558 consensus_peers.cc:487] T 00000000000000000000000000000000 P 4217e682ec1a42c29d70ff8fbe35fa11 -> Peer 8ed9c050eae24bb092ed3dbeb4abade0 (127.0.93.190:43269): Couldn't send request to peer 8ed9c050eae24bb092ed3dbeb4abade0. Status: Network error: Client connection negotiation failed: client connection to 127.0.93.190:43269: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
05:30:14.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260621 05:30:14.822465 557 connection.cc:535] client connection to 127.0.93.189:39347 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
05:30:14.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260621 05:30:14.822932 374 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskVvIgJ7/build/tsan/bin/kudu with pid 531
05:30:15.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-21T05:30:15Z chronyd exiting
05:30:17.010 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29506), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [272ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [565ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [584ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [585ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [588ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [593ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [626ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [644ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [685ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [702ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [703ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [781ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [786ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [788ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [789ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [796ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [802ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [806ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [916ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [926ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [928ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [929ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1037ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [1039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1046ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1476ms] refreshing cache from master, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [1477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [1478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1483ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [2636ms] refreshing cache from master, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [2638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [2642ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [6516ms] refreshing cache from master, [6516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [6516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [6517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [6518ms] too many traces: truncated at 100 traces, deferred=Deferred@1419081759(state=PENDING, result=null, callback=(continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614), errback=(continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295) ~[netty-handler-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290) ~[netty-codec-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428) ~[main/:?]
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762) ~[main/:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.62:37487
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) ~[?:1.8.0_362]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716) ~[?:1.8.0_362]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362]
... 1 more
org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ListTabletServers, tablet=null, attempt=25, TimeoutTracker(timeout=30000, elapsed=29506), Traces: [0ms] refreshing cache from master, [50ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [134ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [138ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [187ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [272ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [565ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [570ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [576ms] refreshing cache from master, [577ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [579ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [584ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [585ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [588ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [591ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [593ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [616ms] refreshing cache from master, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [617ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [618ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [621ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [624ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [625ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [626ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [636ms] refreshing cache from master, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [637ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [639ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [642ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [643ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [644ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [676ms] refreshing cache from master, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [677ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [678ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [681ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [683ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [684ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [685ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [696ms] refreshing cache from master, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [696ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [697ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [699ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [702ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [703ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [705ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [776ms] refreshing cache from master, [776ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [777ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [778ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [781ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [786ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [788ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [789ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [796ms] refreshing cache from master, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [797ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [798ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [801ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [802ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [805ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [806ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [916ms] refreshing cache from master, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [916ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [917ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [922ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [926ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [928ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [929ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1037ms] refreshing cache from master, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [1037ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [1039ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [1042ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [1043ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [1044ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1046ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1476ms] refreshing cache from master, [1476ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [1477ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [1478ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [1480ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [1482ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [1483ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [2636ms] refreshing cache from master, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [2636ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [2638ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.61:45551: Network error: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551, [2641ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.60:36891: OK, [2642ms] Sub RPC ConnectToMaster: received response from server master-127.1.19.62:37487: Network error: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [2642ms] delaying RPC due to: Service unavailable: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487, [6516ms] refreshing cache from master, [6516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.60:36891, [6516ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.61:45551, [6517ms] Sub RPC ConnectToMaster: sending RPC to server master-127.1.19.62:37487, [6518ms] too many traces: truncated at 100 traces, deferred=Deferred@1419081759(state=PENDING, result=null, callback=(continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614), errback=(continuation of Deferred@952922256 after retry RPC after error@1138781154) -> (continuation of Deferred@2142087907 after retry RPC after error@483465) -> (continuation of Deferred@1528092905 after retry RPC after error@468391698) -> (continuation of Deferred@159603774 after retry RPC after error@851142319) -> (continuation of Deferred@1335178113 after retry RPC after error@210104038) -> (continuation of Deferred@527704548 after retry RPC after error@1989386939) -> (continuation of Deferred@276598483 after retry RPC after error@867016932) -> (continuation of Deferred@233364596 after retry RPC after error@1711612168) -> (continuation of Deferred@318451981 after retry RPC after error@1276053814) -> (continuation of Deferred@122644415 after retry RPC after error@1208818495) -> (continuation of Deferred@53815402 after retry RPC after error@935839619) -> (continuation of Deferred@1263086683 after retry RPC after error@1799160558) -> (continuation of Deferred@1096876443 after retry RPC after error@1217605099) -> (continuation of Deferred@1245379414 after retry RPC after error@1234418317) -> (continuation of Deferred@11995133 after retry RPC after error@1369083509) -> (continuation of Deferred@1281834843 after retry RPC after error@418689928) -> (continuation of Deferred@549771140 after retry RPC after error@623381711) -> (continuation of Deferred@1346396886 after retry RPC after error@1838543781) -> (continuation of Deferred@1802199396 after retry RPC after error@247486130) -> (continuation of Deferred@1573241710 after retry RPC after error@529468328) -> (continuation of Deferred@803158396 after retry RPC after error@589733875) -> (continuation of Deferred@1342961226 after retry RPC after error@926840232) -> (continuation of Deferred@1635565290 after retry RPC after error@1991404593) -> (continuation of Deferred@2038725899 after retry RPC after error@1037288614)))
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AsyncKuduClient.tooManyAttemptsOrTimeout(AsyncKuduClient.java:1889)
at org.apache.kudu.client.AsyncKuduClient.delayedSendRpcToTablet(AsyncKuduClient.java:2337)
at org.apache.kudu.client.AsyncKuduClient.access$1500(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1520)
at org.apache.kudu.client.AsyncKuduClient$RetryRpcErrback.call(AsyncKuduClient.java:1510)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:313)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.NoLeaderFoundException: Master config (127.1.19.60:36891,127.1.19.61:45551,127.1.19.62:37487) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.61:45551(127.1.19.61:45551): Connection refused: /127.1.19.61:45551,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:311)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:338)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterCB.call(ConnectToCluster.java:327)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.callback(KuduRpc.java:324)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:321)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.channelRead0(Connection.java:377)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1383)
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1246)
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1295)
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.1.19.62:37487(127.1.19.62:37487): Connection refused: /127.1.19.62:37487
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:428)
at org.apache.kudu.client.Connection$1.operationComplete(Connection.java:762)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629)
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
... 8 more
Caused by: io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: /127.1.19.62:37487
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:337)
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:750)